diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index bfe85cbb..4b86eec0 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -2,16 +2,17 @@ # INTEGR_TEST_* secrets; where they are unavailable (e.g. forks), the # credential fixtures make the tests skip instead of fail. # -# Runs on every pull request into main, so breaking changes surface before the -# release job (gated separately by required review on the pypi environment) -# ever starts. No push trigger: a push to main is already covered by the PR -# that preceded it, so a second run would add nothing. +# Runs on every pull request into main or stage, so breaking changes surface +# before the release job (gated separately by required review on the pypi +# environment) ever starts. stage collects reviewed pull requests before they +# go to main together. No push trigger: a push to main is already covered by +# the PR that preceded it, so a second run would add nothing. name: Integration on: pull_request: - branches: [main] + branches: [main, stage] workflow_dispatch: concurrency: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index da26db27..2548d777 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,7 +2,7 @@ name: Main on: push: - branches: [main] + branches: [main, stage] pull_request: types: [opened, synchronize, reopened, ready_for_review] workflow_dispatch: diff --git a/docs/dev.md b/docs/dev.md index 82625fce..0d7e0540 100644 --- a/docs/dev.md +++ b/docs/dev.md @@ -66,8 +66,8 @@ excluded by default: Integration tests run against a live wiki and need credentials; tests whose credentials are missing skip instead of fail. CI runs them on every -pull request into `main` (workflow `integration.yml`), so they gate the -release rather than run after it. +pull request into `main` or `stage` (workflow `integration.yml`), so they +gate the release rather than run after it. === "make" diff --git a/docs/tools/index.md b/docs/tools/index.md index 45dd54c3..0c303199 100644 --- a/docs/tools/index.md +++ b/docs/tools/index.md @@ -2,8 +2,10 @@ Besides the Python API, osw ships two adapters that talk to a live instance: the `osw` command line client, and an MCP server for agent clients such as -Claude Code. Both run the same operations from one shared, SDK-free core -(`osw.service`), so a command and its matching tool behave identically. They +Claude Code. This section also covers standalone scripts built on the Python +API. The current one is the [user sync](user-sync.md). The two adapters run +the same operations from one shared, SDK-free core (`osw.service`), so a +command and its matching tool behave identically. They differ in exactly one way: only the CLI accepts filesystem paths. ## Setup @@ -49,4 +51,5 @@ extras are listed in the [Get Started guide](../get-started.md#optional-extras). | ---- | -------- | | [CLI](cli.md) | The `.env` quick start, the full command reference, and the global flags | | [MCP server](mcp.md) | The tool surface, the no-filesystem-access and one-server-per-instance rules, and how to register the server with a client such as Claude Code | +| [User Sync (script)](user-sync.md) | A Python script that creates or updates `User` items from the MediaWiki accounts of an instance, with the options, the data it writes and how it reconciles existing items | | [Configuration](configuration.md) | What both adapters share: where credentials and settings come from, and the full environment-variable reference | diff --git a/docs/tools/user-sync.md b/docs/tools/user-sync.md new file mode 100644 index 00000000..6d016f66 --- /dev/null +++ b/docs/tools/user-sync.md @@ -0,0 +1,112 @@ +# User Item Sync + +Create or update OSW `User` items from the MediaWiki accounts of an OSL +instance, enriching ORCID users from the public ORCID API. The run is +idempotent and previews every change before writing. + +This is a Python script built on the osw library. It is not a command of the +`osw` CLI and not an MCP tool, see [Tools](index.md). It runs from a source +checkout, because `examples/` is not part of the installed package. + +## When to use it + +OSL instances gain accounts from manual creation and from whitelisted ORCID +single sign-on (whose username is the ORCID iD). Those accounts have no +semantic `User` item until this tool reconciles them. + +## Run it + +```bash +uv run python examples/user_sync.py --domain your-instance.example.org --dry-run +``` + +The script reads the credential file given by `--cred-filepath`, otherwise +`accounts.pwd.yaml` in the working directory, otherwise the file named by the +`OSW_CRED_FILEPATH` environment variable. It does not read a `.env` file. +`--domain` falls back to the `OSW_DOMAIN` environment variable. See +[Credentials](configuration.md#credentials) for the file format. Start with +`--dry-run` to preview, then run without it to write after confirming at the +prompt. + +To also clean up existing items, add `--prune`. Without it, the sync never +writes `employment_contract_status` and never removes anything. With it, +`employment_contract_status` is removed from the stored jsondata of existing +items. `website` and `organization` are overwritten with an empty list, unless +`--with-websites` or `--with-organizations` enables them. This happens in a +second edit of the page's jsondata after the store. The sync decides from the +stored jsondata, not from the loaded item, which shows the model default for +`employment_contract_status`. A second `--prune` run therefore changes nothing: + +```bash +uv run python examples/user_sync.py --domain your-instance.example.org --prune --dry-run +``` + +Users whose ORCID record exposes no email are listed in a warning at the end of +the run; a missing email never fails the sync. + +### Options + +By default the tool syncs both ORCID and MediaWiki-native accounts and writes +only core identity plus email. + +| Flag | Effect | +| --- | --- | +| `--domain` | Target OSL domain. | +| `--cred-filepath` | Path to `accounts.pwd.yaml`. | +| `--dry-run` | Preview only; never writes. | +| `--auto-apply` | Non-interactive: apply creates, gap-fills and removals; keep existing on conflicts. | +| `--limit N` | Process at most N accounts (testing). | +| `--orcid-only` | Only ORCID-username accounts (excludes `--mw-only`). | +| `--mw-only` | Only non-ORCID (MediaWiki-native) accounts (excludes `--orcid-only`). | +| `--include-system` | Do not skip MediaWiki reserved system accounts. | +| `--no-redirects` | Do not create `User:` redirect pages. | +| `--with-websites` | Also store ORCID researcher URLs (opt-in). | +| `--with-organizations` | Also create and link Organization items from ORCID affiliations (opt-in). | +| `--with-extras` | Enable both websites and organizations. | +| `--include-blocked` | Do not skip blocked MediaWiki accounts. | +| `--prune` | Remove `employment_contract_status` from the stored jsondata of existing items, and overwrite `website` or `organization` with an empty list when their opt-in flag is not given, in a second edit of the page's jsondata after the store (default off; a normal run never writes `employment_contract_status` and never removes anything). | + +## What it does + +1. Enumerate MediaWiki accounts, skipping blocked accounts (unless + `--include-blocked` is set; the User item of a skipped blocked account is + left unchanged), bots and privileged groups + (`bot`, `sysop`, `bureaucrat`, `interface-admin`) and MediaWiki reserved + system usernames. Accounts whose username is an exact ORCID iD are exempt + from the group filter (so a person's own admin account is synced); other + accounts, such as `@bot`, are still excluded by their groups. +2. Enrich ORCID users from `https://pub.orcid.org/v3.0/`: names and email + always; researcher URLs and affiliations only when their flag is enabled. +3. Map each account to a proposed `User` item with a deterministic id + (uuid5 on the ORCID iD, else the username), so re-runs are idempotent. +4. Reconcile against existing items. Existing items are found by `username` + across all `User` items. An item whose deterministic page already exists is + updated and not created again. Each user gets one of four categories: NEW, + GAP_FILL, CONFLICT or UNCHANGED. Each changed field has one of three + statuses: GAP_FILL, CONFLICT or REMOVE. The preview shows a removed field as + `-field`. A user with only gap-fills or removals is GAP_FILL. Gap-fills apply + automatically and conflicts are resolved interactively. Removals appear only + with `--prune`; a normal run never removes anything. +5. Store items, create `User:` redirects to each item, ensure any + linked Organization items, verify by reloading, and warn about users with no + email. + +## Data written + +| Field | Policy | +| --- | --- | +| `username`, `first_name`, `surname`, `label`, `orcid` | Always (core identity). | +| `email` | Standard: always attempted; missing email is warned, not fatal; never removed. | +| `website` | Opt-in (`--with-websites`); without that flag, on existing items overwritten with an empty list only with `--prune`, in a second jsondata edit after the store. | +| `organization` | Opt-in (`--with-organizations`); without that flag, on existing items overwritten with an empty list only with `--prune`, in a second jsondata edit after the store. | +| `employment_contract_status` | Never written without `--prune`; with `--prune`, removed from the stored jsondata of existing items in a second jsondata edit after the store. | + +## Notes + +- MediaWiki does not expose other users' email or real name, so ORCID is the + only source of rich data. Non-ORCID accounts get username-based placeholder + names, flagged in the preview. Placeholder names are written only for new + items and are never proposed for an existing item. +- In a source checkout (for developers), the first accepted run autofetches + schemas and may regenerate `src/osw/model/entity.py`. Restore it if you do not + want that change. diff --git a/examples/user_sync.py b/examples/user_sync.py new file mode 100644 index 00000000..9e507d74 --- /dev/null +++ b/examples/user_sync.py @@ -0,0 +1,30 @@ +"""Create or update OSW User items from MediaWiki accounts and ORCID. + +Usage: + uv run python examples/user_sync.py --domain llm4eln.semos.dev --dry-run +""" + +from osw.defaults import paths +from osw.express import OswExpress +from osw.tools.user_sync import config_from_args, run_user_sync + + +def main() -> None: + config = config_from_args() + cred_filepath = config.cred_filepath + if cred_filepath is None and paths.cred_filepath.is_file(): + # OswExpress only reads OSW_CRED_FILEPATH; use the default accounts file. + cred_filepath = paths.cred_filepath + osw = OswExpress(domain=config.domain, cred_filepath=cred_filepath) + report = run_user_sync(config, osw=osw) + print(report.summary()) + for title in report.created: + print(f" created: {title}") + for title in report.updated: + print(f" updated: {title}") + for key, error in report.failed.items(): + print(f" FAILED {key}: {error}") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 51f2e253..e5b462b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,9 +21,11 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", ] dependencies = [ - "oold>=0.15.0", - "opensemantic", - "opensemantic.core>=0.57.4", + # registered_credentials(), used to find the wiki of a file without a URL + "oold>=2.3.0", + "opensemantic>=0.2.5", + # the file controllers this module wraps + "opensemantic.core>=0.59.0", "opensemantic.base>=0.42.7", "pydantic[email]>=2.12.0", "datamodel-code-generator==0.51.0", diff --git a/src/osw/cli/ops.py b/src/osw/cli/ops.py index 62cf9669..f91cd6cb 100644 --- a/src/osw/cli/ops.py +++ b/src/osw/cli/ops.py @@ -271,7 +271,7 @@ def status_instances(ctx: Context) -> dict: # Not ctx.osw: that one is cached and bound to the active # instance. These connections are ours alone, so we close each # one instead of leaving it open for the rest of the command. - connection = ctx.osw_for(config.derive_domain(iri)) + connection = ctx.osw_for(config.derive_connect_domain(iri)) entry["connected"] = True _close_quietly(connection, iri) except Exception as exc: diff --git a/src/osw/controller/file/__init__.py b/src/osw/controller/file/__init__.py index e69de29b..53e9c5e1 100644 --- a/src/osw/controller/file/__init__.py +++ b/src/osw/controller/file/__init__.py @@ -0,0 +1,10 @@ +"""File controllers. + +Imported eagerly so the controllers register themselves: a caller that resolves +a file entity through the controller registry then finds the wiki +implementation without importing it by name. +""" + +from osw.controller.file import base, local, memory, remote, wiki + +__all__ = ["base", "local", "memory", "remote", "wiki"] diff --git a/src/osw/controller/file/base.py b/src/osw/controller/file/base.py index 3405cf59..22b531f4 100644 --- a/src/osw/controller/file/base.py +++ b/src/osw/controller/file/base.py @@ -1,66 +1,9 @@ -from abc import abstractmethod -from typing import IO, Any, Dict +"""File controller interface. -from osw.core import model +The implementation moved to ``opensemantic.core``, which owns the file models. +This module keeps the import path working. +""" +from opensemantic.core.v1 import FileControllerMixin as FileController -class FileController(model.File): - """Base class for file controllers""" - - @abstractmethod - def get(self) -> IO: - """Reads or downloads a file from an IO object - Make sure to close the IO object after reading - - Returns - ------- - the IO object of the opened file - """ - - @abstractmethod - def put(self, file: IO, **kwargs: Dict[str, Any]): - """Writes or uploads a file from an IO object - - Parameters - ------- - file - the IO object to read from - """ - - def put_from(self, other: "FileController", **kwargs: Dict[str, Any]): - """Writes or uploads a file from another file controller - Automatically closes the other file controllers IO object after reading - - Parameters - ---------- - other - another file controller to read from - """ - with other.get() as f: - self.put(f, **kwargs) - - def get_to(self, other: "FileController"): - """Reads or downloads a file to another file controller - Automatically closes this file controllers IO object after reading - - Parameters - ---------- - other - another file controller to write to - """ - with self.get() as f: - other.put(f) - - @classmethod - def from_other(cls, other: "FileController", **kwargs: Any) -> "FileController": - """Create an instance based on another file controller - copying all common attributes - - Parameters - ---------- - other - another file controller to copy from - kwargs - additional attributes to be set - """ - return other.cast(cls, **kwargs) +__all__ = ["FileController"] diff --git a/src/osw/controller/file/local.py b/src/osw/controller/file/local.py index 17137e46..fe943802 100644 --- a/src/osw/controller/file/local.py +++ b/src/osw/controller/file/local.py @@ -1,58 +1,68 @@ +"""Local file controller. + +The implementation moved to ``opensemantic.core``, which owns the file models. +This subclass keeps the ``path`` argument and the filename-derived label of the +controller it replaces. +""" + import os -import shutil -from io import TextIOBase from pathlib import Path -from typing import IO, Any, Dict, List, Optional +from typing import IO, Any, Dict, Optional + +from opensemantic.core.v1 import LocalFileController as _LocalFileController -from osw.controller.file.base import FileController from osw.core import model -class LocalFileController(FileController, model.LocalFile): +class LocalFileController(_LocalFileController): """File controller for local files""" - label: Optional[List[model.Label]] = [model.Label(text="Unnamed file")] - """The label of the file, defaults to 'Unnamed file'""" - path: Path - """The path to the file""" + path: Optional[Path] = None + """Where the content lives, as a Path. A field, not a property: a property + of this name would win over the ``path`` field declared by + ``osw.express.FileResult`` and make that field's value unreadable. Kept in + step with the model's own ``local_path`` by the constructor, so either name + reads the same file.""" class Config: arbitrary_types_allowed = True - @classmethod - def from_other( - cls, other: "FileController", path: Path, **kwargs - ) -> "LocalFileController": - return other.cast(cls, path=path, **kwargs) - def __init__(self, **kwargs): + path = kwargs.get("path") + if path is not None and not kwargs.get("local_path"): + kwargs["local_path"] = str(path) super().__init__(**kwargs) + if not self.local_path: + # The controller this replaces declared path as a required field, + # so an instance without one never existed. Raising here keeps the + # error at the call site that is wrong, instead of surfacing it on + # the first read. + raise ValueError( + f"{type(self).__name__} needs a file: pass path= or local_path=" + ) + self.path = Path(self.local_path) self._set_metadata() + @classmethod + def from_other(cls, other, path=None, **kwargs) -> "LocalFileController": + if path is not None: + kwargs["local_path"] = str(path) + return other.cast(cls, **kwargs) + def get(self) -> IO: self._set_metadata() - return open(self.path, "rb") + return super().get() def put(self, file: IO, **kwargs: Dict[str, Any]): self._set_metadata() - mode = "wb" - if isinstance(file, TextIOBase): - mode = "w" - with open(self.path, mode) as f: - # f.write(file.read()) # In-memory - limited by available RAM - shutil.copyfileobj(file, f) # Chunked copy - - def delete(self): - if os.path.exists(self.path): - os.remove(self.path) + super().put(file, **kwargs) def _set_metadata(self): - file = LocalFileController.extract_metadata(self.path) - self.label = file.label + self.label = LocalFileController.extract_metadata(self.path).label @classmethod def extract_metadata(cls, path: Path) -> model.LocalFile: - file = model.LocalFile(label=[model.Label(text=path.name)]) + file = model.LocalFile(label=[model.Label(text=Path(path).name)]) if os.path.exists(path): pass # stats = os.stat(path) diff --git a/src/osw/controller/file/memory.py b/src/osw/controller/file/memory.py index 5bd96583..aea72dc8 100644 --- a/src/osw/controller/file/memory.py +++ b/src/osw/controller/file/memory.py @@ -1,28 +1,9 @@ -import shutil -from io import BytesIO -from typing import IO, Any, Dict, List, Optional +"""In-memory file controller. -from pydantic.v1 import Field +The implementation moved to ``opensemantic.core``, which owns the file models. +This module keeps the import path working. +""" -from osw.controller.file.base import FileController -from osw.core import model +from opensemantic.core.v1 import InMemoryController - -class InMemoryController(FileController, model.LocalFile): - """File controller for in-memory streams""" - - label: Optional[List[model.Label]] = [model.Label(text="Unnamed stream")] - """the label of the stream, e.g., the name of the file the stream - originates from. Defaults to 'Unnamed stream'.""" - stream: Any = Field(default_factory=BytesIO) - """the stream to the file, any file-like object. Defaults to an empty - binary buffer. Byte-oriented, to match the get/put counterparts.""" - - class Config: - arbitrary_types_allowed = True - - def get(self) -> IO: - return self.stream - - def put(self, file: IO, **kwargs: Dict[str, Any]): - shutil.copyfileobj(file, self.stream) +__all__ = ["InMemoryController"] diff --git a/src/osw/controller/file/remote.py b/src/osw/controller/file/remote.py index 11a592ea..e9bf1ee9 100644 --- a/src/osw/controller/file/remote.py +++ b/src/osw/controller/file/remote.py @@ -1,31 +1,9 @@ -from abc import abstractmethod -from typing import IO, Any, Dict +"""Remote file controller. -from osw.controller.file.base import FileController -from osw.core import model +The implementation moved to ``opensemantic.core``, which owns the file models. +This module keeps the import path working. +""" -# TODO: add additional remove file with -# https://docs.prefect.io/2.11.4/concepts/filesystems/ +from opensemantic.core.v1 import RemoteFileController - -# Note: the order of the base classes is important -# The data class must be the first base class, otherwise subclass controllers fall back -# to the data model of the controller superclass -class RemoteFileController(model.RemoteFile, FileController): - @abstractmethod - def get(self) -> IO: - pass - - @abstractmethod - def put(self, file: IO, **kwargs: Dict[str, Any]): - pass - - # @classmethod - # def from_local(self, local: "local.LocalFileController") -> "RemoteFileController": - # rf = local.cast(self) - # rf.put(local.get()) - # return rf - - # def from_local(self, local: "local.LocalFileController") -> None: - # with local.get() as f: - # self.put(f) +__all__ = ["RemoteFileController"] diff --git a/src/osw/controller/file/wiki.py b/src/osw/controller/file/wiki.py index 88139d79..7cd0f9e1 100644 --- a/src/osw/controller/file/wiki.py +++ b/src/osw/controller/file/wiki.py @@ -1,10 +1,14 @@ import functools import logging import os -from typing import IO, Any, Dict, List, Optional +from typing import IO, Any, Dict, List, Optional, Tuple +from urllib.parse import urlparse import mwclient.errors +from oold.backend.auth import get_credential as _get_registered_credential +from oold.backend.auth import registered_credentials as _registered_credentials +from osw.auth import CredentialManager from osw.controller.file.base import FileController from osw.controller.file.remote import RemoteFileController from osw.core import OSW, model @@ -79,13 +83,72 @@ def store_params_for_upload( return {**se_params, "overwrite": "replace remote"} +_sessions: Dict[Tuple[str, str], Tuple[Any, OSW]] = {} +"""Sessions by (host, credential IRI), with the credential they were built from. + +The credential is part of the key because two credentials scoped to different +paths of one wiki are two different logins; the host is, because one credential +can match several hosts - oold's lookup treats a stored IRI contained in the +searched one as a match, so a credential for ``example.org`` answers for every +subdomain. The credential itself is kept so a later ``set_credential`` replaces +the session instead of being ignored for the lifetime of the process. +""" + + +def clear_sessions() -> None: + """Drop the cached sessions, so the next access logs in again.""" + _sessions.clear() + + +def host_of(url: str) -> str: + """The wiki host a URL or a bare credential IRI names. + + A credential is registered under whatever its owner chose - a bare host, a + host with a path, a full URL. ``WtSite`` wants the host alone: given a path + it builds an API endpoint that does not exist. + """ + parsed = urlparse(url if "//" in url else f"//{url}") + return parsed.netloc or url.split("/")[0] + + +def credential_for(url: str) -> Any: + """The credential registered for ``url``, or a readable error.""" + try: + return _get_registered_credential(url) + except ValueError as error: + raise ValueError( + f"No credential registered for {host_of(url)}. Pass osw= explicitly " + "or register one with oold.backend.auth.set_credential." + ) from error + + +def session_for(url: str) -> OSW: + """An OSW session for the host of ``url``, from the registered credential. + + The file entity may be stored somewhere other than its bytes, so the + lookup key is the URL the content lives at, not the entity's own IRI. + """ + host = host_of(url) + credential = credential_for(url) + key = (host, credential.iri) + cached = _sessions.get(key) + if cached is not None and cached[0] == credential: + return cached[1] + cred_mngr = CredentialManager() + cred_mngr.add_credential(credential) + session = OSW(site=WtSite(WtSite.WtSiteConfig(iri=host, cred_mngr=cred_mngr))) + _sessions[key] = (credential, session) + return session + + class WikiFileController(model.WikiFile, RemoteFileController): """File controller for wiki files""" label: Optional[List[model.Label]] = [model.Label(text="Unnamed file")] """the label of the file, defaults to 'Unnamed file'""" - osw: OSW - """the OSW instance to connect with""" + osw: Optional[OSW] = None + """the OSW instance to connect with. Left unset, it is created on first + use from the credential registered for download_url / access_url.""" namespace: Optional[str] = "File" """the namespace of the file, defaults to 'File'""" title: Optional[str] = None @@ -98,9 +161,17 @@ class Config: @classmethod def from_other( - cls, other: FileController, osw: OSW, **kwargs: Dict[str, Any] + cls, + other: FileController, + osw: Optional[OSW] = None, + **kwargs: Dict[str, Any], ) -> "WikiFileController": - return super().from_other(other, osw=osw, **kwargs) + # Only pass a session that was given: writing None over the one the + # source already carries would defer the failure to put time, with a + # message about a missing credential rather than a dropped session. + if osw is not None: + kwargs["osw"] = osw + return super().from_other(other, **kwargs) def get(self) -> IO: self._init() @@ -177,6 +248,7 @@ def get_to(self, other: "FileController"): other.put(response.raw) def put(self, file: IO, **kwargs: Dict[str, Any]): + self._ensure_session() # extract file meta information if hasattr(file, "name") and file.name is not None: name = os.path.basename(file.name) @@ -235,6 +307,7 @@ def put_from(self, other: FileController, **kwargs: Dict[str, Any]): super().put_from(other, **kwargs) def delete(self): + self._ensure_session() file_page_name = f"{self.namespace}:{self.title}" file_page = self.osw.site.get_page( WtSite.GetPageParam(titles=[file_page_name]) @@ -244,12 +317,47 @@ def delete(self): @property def url(self): + self._ensure_session() return ( f"{self.osw.mw_site.scheme}://{self.osw.mw_site.host}" f"/wiki/File:{self.title}" ) + def _ensure_session(self) -> OSW: + """The session, connecting on first use when none was passed in. + + A file that is about to be uploaded has no URL yet - that is what the + upload produces - so the URL is only one of the ways to find the wiki. + Without one the registered credentials decide, and they are only + unambiguous when there is exactly one. + """ + if self.osw is None: + self.osw = session_for(self._session_url()) + return self.osw + + def _session_url(self) -> str: + """The wiki to connect to: the file's own URL, else the credential's.""" + try: + return self._url() + except ValueError: + pass + registered = _registered_credentials() + if len(registered) == 1: + return next(iter(registered)) + if not registered: + raise ValueError( + f"{type(self).__name__} has no download_url or access_url and " + "no credential is registered. Pass osw= explicitly or register " + "one with oold.backend.auth.set_credential." + ) + raise ValueError( + f"{type(self).__name__} has no download_url or access_url, and " + f"credentials are registered for {', '.join(sorted(registered))}. " + "Pass osw= explicitly to say which wiki this file belongs to." + ) + def _init(self, name=None, suffix=None): + self._ensure_session() # set the name attribute to the actual file name, e.g., "image.png" if self.name is None: self.name = name diff --git a/src/osw/core.py b/src/osw/core.py index 03594f6b..55605129 100644 --- a/src/osw/core.py +++ b/src/osw/core.py @@ -32,6 +32,7 @@ set_resolver, ) from oold.generator import Generator +from oold.model.v1 import _types as oold_type_registry from oold.utils.codegen import OOLDJsonSchemaParser from opensemantic.v1 import OswBaseModel from pydantic import PydanticDeprecatedSince20 @@ -39,15 +40,6 @@ from pyld import jsonld import osw.model.entity as model - -try: - # Characteristics of the released opensemantic packages derive from this - # class, not from the one in osw.model.entity. - from opensemantic.core.v1 import ( - CharacteristicType as _OpensemanticCharacteristicType, - ) -except ImportError: # pragma: no cover - optional dependency - _OpensemanticCharacteristicType = None from osw.defaults import params as default_params from osw.utils.code_postprocessing import ( remove_constraints_from_forward_refs, @@ -58,6 +50,7 @@ AggregateGeneratedSchemasParamMode, aggregate_generated_schemas, escape_json_strings, + merge_deep, merge_generated_definitions, ) from osw.utils.templates import ( @@ -68,6 +61,7 @@ from osw.utils.wiki import ( get_full_title, get_namespace, + get_osw_id, get_title, is_empty, namespace_from_full_title, @@ -210,6 +204,258 @@ def read_file_if_exists(path: str) -> str: return f.read() +def remove_generated_schema_section( + schema: dict, def_key: str = "$defs", gen_def_key: str = "generated" +) -> dict: + """Removes the generated section and the allOf entry that points to it. + + aggregate_generated_schemas() merges into an existing generated section and + merge_deep() concatenates arrays, so a stale 'required' entry or 'type' would + survive every regeneration. Mirrors osl.util.postProcessPage in PageBot.js of + mediawiki-extensions-OpenSemanticLab. Modifies schema in place. + """ + gen_def_pointer = f"#/{def_key}/{gen_def_key}" + defs = schema.get(def_key) + if isinstance(defs, dict): + defs.pop(gen_def_key, None) + if isinstance(schema.get("allOf"), list): + schema["allOf"] = [ + entry + for entry in schema["allOf"] + if not (isinstance(entry, dict) and entry.get("$ref") == gen_def_pointer) + ] + return schema + + +CATEGORY_CATEGORY_UUID = "89aafe6d-ae5a-4f29-97ff-df7736d4cab6" +CATEGORY_ENTITY_UUID = "ce353767-c628-45bd-9d88-d6eb3009aec0" +# the definitions section mode is selected when this template is in the list +CHARACTERISTIC_TYPE_TITLE = "Category:OSWffe74f291d354037b318c422591c5023" + + +def _title_from_schema_ref(ref: Any) -> Optional[str]: + """Page title of a $ref to a category schema, None for any other $ref. + + Accepts '/wiki/Category:X?action=raw&slot=jsonschema' and + '.../Special:SlotResolver/Category/X.slot_jsonschema.json'. + """ + if not isinstance(ref, str) or ref.startswith("#"): + return None + path = ref.split("?")[0] + match = re.search(r"/Special:SlotResolver/([^/]+)/(.+)\.slot_[a-z]+\.json$", path) + if match: + return f"{match.group(1)}:{match.group(2)}" + match = re.search(r"/wiki/(Category:[^/]+)$", path) + if match: + return match.group(1) + if re.fullmatch(r"Category:[^/]+", path): + return path + return None + + +def collect_category_template_titles( + get_pages, meta_category_titles: List[str] +) -> List[str]: + """Titles of the category templates the wiki form editor applies. + + Mirrors the editor: the schema of the meta categories is bundled, the uuid of + every schema is collected after the schemas below it (allOf, oneOf, anyOf, + definitions, $defs, every entry of properties and the items of each property + are followed in this order, so the most generic schema comes first) and the + result starts with Category:Category. A referenced schema is followed only + at its first occurrence, like in a bundled schema. + Category:Category and Category:Entity are not collected. Follows + osl.util.postProcessPage (PageBot.js) with editor.jsonschema.subschemas_uuids + (MwJson_schema.js, _preprocess) in mediawiki-extensions-OpenSemanticLab and + mediawiki-extensions-MwJson. A '/wiki/?action=raw' reference is + followed only for category pages, e.g. '/wiki/JsonSchema:Label?action=raw' + is not followed. A 'Special:SlotResolver' reference is followed for any page. + + Parameters + ---------- + get_pages + callable that returns the page objects for a list of titles + meta_category_titles + titles of the meta categories, e.g. ["Category:Category"] + """ + templates = ["Category:Category"] + schema_cache: Dict[str, dict] = {} + visited = set() + + def schema_of(title: str) -> dict: + if title not in schema_cache: + schema = {} + for page in get_pages([title]): + if page.title == title: + schema = page.get_slot_content("jsonschema") or {} + if not schema: + _logger.warning( + f"No jsonschema found for {title}. Its templates are not " + "applied, which can select the wrong schema generation mode." + ) + schema_cache[title] = schema + return schema_cache[title] + + def walk(schema: Any) -> None: + if not isinstance(schema, dict): + return + ref_title = _title_from_schema_ref(schema.get("$ref")) + if ref_title is not None: + # a bundled schema is included once, later refs point to it + if ref_title not in visited: + visited.add(ref_title) + walk(schema_of(ref_title)) + return + for key in ("allOf", "oneOf", "anyOf"): + sub = schema.get(key) + for entry in sub if isinstance(sub, list) else [sub]: + walk(entry) + for key in ("definitions", "$defs"): + if isinstance(schema.get(key), dict): + for entry in schema[key].values(): + walk(entry) + if isinstance(schema.get("properties"), dict): + for entry in schema["properties"].values(): + walk(entry) + if isinstance(entry, dict): + walk(entry.get("items")) + uuid = schema.get("uuid") + if uuid and uuid not in (CATEGORY_CATEGORY_UUID, CATEGORY_ENTITY_UUID): + title = "Category:" + get_osw_id(uuid) + if title not in templates: + templates.append(title) + + for title in meta_category_titles: + visited.add(title) + walk(schema_of(title)) + return templates + + +def reset_generated_jsonld_context( + schema: Optional[dict], had_generated_section: bool +) -> None: + """Empties the generated first object block of the @context array. + + Port of osl.util.resetGeneratedJsonLdContext in PageBot.js. The generated + property mappings are by convention the first object entry of an @context + array, the strings before it are parent schema references. Without a generated + section the first object block is hand written, so an empty block is inserted + ahead of it. A string or object @context is left as it is. Modifies schema in + place. + """ + if not schema: + return + context = schema.get("@context") + if not isinstance(context, list): + return + index = next((i for i, e in enumerate(context) if isinstance(e, dict)), -1) + if not had_generated_section: + context.insert(len(context) if index == -1 else index, {}) + return + if index != -1: + context[index] = {} + else: + context.append({}) + + +def merge_generated_jsonld_context(schema: dict, generated_context: Any) -> None: + """Merges the @context of a generated schema into the schema. + + Port of the @context handling of osl.util.postProcessPage in PageBot.js. The + generated mappings are merged into the first object block of the @context + array only, which reset_generated_jsonld_context() reserved. oold's + aggregate_generated_schemas() collapses all object blocks into one, which + would fuse a hand written block with the generated one. String entries + (parent schema references) of the generated context are appended when new. + Modifies schema in place. + """ + existing = schema.get("@context") + if isinstance(existing, list): + context_list = existing + else: + context_list = [existing] if existing else [] + entries = ( + generated_context + if isinstance(generated_context, list) + else [generated_context] + ) + generated_object: dict = {} + for entry in entries: + if isinstance(entry, dict): + generated_object = merge_deep(generated_object, entry) + elif entry and entry not in context_list: + context_list.append(entry) + index = next((i for i, e in enumerate(context_list) if isinstance(e, dict)), -1) + if index == -1: + context_list.append(generated_object) + else: + context_list[index] = merge_deep(context_list[index], generated_object) + schema["@context"] = context_list + + +def strip_root_members_superseded_by_generated( + root_schema: dict, generated_schema: dict +) -> bool: + """Removes the root members the generated section defines. + + Port of osl.util.stripRootMembersSupersededByGenerated in PageBot.js. A member + declared at root takes precedence over the same member pulled in through + allOf, so a copy of the generated members at root would shadow the freshly + generated ones. Containers are searched recursively, manual siblings stay. + Members that are written to root on purpose are skipped. The allOf entries that + the generated section repeats are removed from root. Modifies root_schema in + place and returns True when anything was removed. + """ + promoted_to_root = { + "title", + "title*", + "description", + "description*", + "@context", + "$defs", + "allOf", + "$ref", + "$comment", + } + removed = False + + def strip(target, generated, is_root): + nonlocal removed + if not isinstance(target, dict) or not isinstance(generated, dict): + return + for key in list(generated.keys()): + if is_root and key in promoted_to_root: + continue + if key not in target: + continue + if isinstance(generated[key], dict) and isinstance(target[key], dict): + strip(target[key], generated[key], False) + if len(target[key]) == 0: + del target[key] + removed = True + else: + del target[key] + removed = True + + strip(root_schema, generated_schema, True) + + generated_allof = generated_schema.get("allOf") + generated_allof = generated_allof if isinstance(generated_allof, list) else [] + if isinstance(root_schema.get("allOf"), list) and generated_allof: + generated_refs = [ + e["$ref"] for e in generated_allof if isinstance(e, dict) and e.get("$ref") + ] + kept = [ + e + for e in root_schema["allOf"] + if not (isinstance(e, dict) and e.get("$ref") in generated_refs) + ] + if len(kept) != len(root_schema["allOf"]): + root_schema["allOf"] = kept + removed = True + return removed + + # Reusable type definitions class OverwriteOptions(Enum): """Options for overwriting properties""" @@ -244,6 +490,46 @@ class AddOverwriteClassOptions(Enum): OVERWRITE_CLASS_OPTIONS = Union[OverwriteOptions, AddOverwriteClassOptions] +def _explicitly_set_empty(entity: OswBaseModel, jsondata: dict) -> dict: + """Returns the top-level jsondata entries the caller set to an empty value. + + remove_empty() cannot tell an empty value that was assigned on purpose from + one that is merely a default, so it strips both. Assigning an empty value is + how a property gets cleared, though: the merge in _apply_overwrite_policy() + starts from the remote content, so a local key that was stripped leaves the + remote value in place and the property can never be emptied. pydantic records + the fields that were explicitly supplied in __fields_set__, which is enough to + keep those and drop the rest. + + Parameters + ---------- + entity: + The entity the jsondata was serialized from + jsondata: + The serialized entity, before remove_empty() has been applied + + Returns + ------- + result: + The subset of jsondata that is empty but was explicitly set + """ + fields_set = getattr(entity, "__fields_set__", None) + if not fields_set: + return {} + fields = getattr(entity, "__fields__", {}) + keys = set() + for name in fields_set: + # json() is called without by_alias, so the key is the field name. Accept + # the alias too, so that a field declared with one (model.PrefectFlow + # names 'schema_' after 'schema') is matched either way. + keys.add(name) + if name in fields: + keys.add(fields[name].alias) + return { + key: value for key, value in jsondata.items() if key in keys and is_empty(value) + } + + class OSW(BaseModel): """Bundles core functionalities of OpenSemanticWorld (OSW)""" @@ -631,24 +917,36 @@ def fetch_schema( first = True last = False results = [] - for schema_title in fetchSchemaParam.schema_title: - last = schema_title == fetchSchemaParam.schema_title[-1] - mode = fetchSchemaParam.mode - if not first: # 'replace' makes only sense for the first schema - mode = "append" - res = self._fetch_schema( - OSW._FetchSchemaParam( - schema_title=schema_title, - mode=mode, - final=last, - generate_annotations=fetchSchemaParam.generate_annotations, - generator_options=fetchSchemaParam.generator_options, - offline_pages=fetchSchemaParam.offline_pages, - result_model_path=fetchSchemaParam.result_model_path, + # the page cache is enabled once for the whole operation, because the same + # schema pages are read repeatedly while $refs are resolved. The state is + # taken and restored here and not per schema title: _fetch_schema would + # snapshot the state its own predecessor has already changed. The restore + # runs in a finally block so that an early return or an exception in + # _fetch_schema cannot leave the cache enabled for the rest of the process. + site_cache_state = self.site.get_cache_enabled() + self.site.enable_cache() + try: + for schema_title in fetchSchemaParam.schema_title: + last = schema_title == fetchSchemaParam.schema_title[-1] + mode = fetchSchemaParam.mode + if not first: # 'replace' makes only sense for the first schema + mode = "append" + res = self._fetch_schema( + OSW._FetchSchemaParam( + schema_title=schema_title, + mode=mode, + final=last, + generate_annotations=fetchSchemaParam.generate_annotations, + generator_options=fetchSchemaParam.generator_options, + offline_pages=fetchSchemaParam.offline_pages, + result_model_path=fetchSchemaParam.result_model_path, + ) ) - ) - results.append(res) - first = False + results.append(res) + first = False + finally: + if not site_cache_state: + self.site.disable_cache() # restore original state # merge unique results and return merged_result = OSW.FetchSchemaResult( @@ -729,9 +1027,13 @@ def _fetch_schema( ---------- fetchSchemaParam See FetchSchemaParam, by default None + + Notes + ----- + The page cache is enabled and restored by the calling fetch_schema(), not + here. This method is called once per schema title and recursively per $ref, + so a snapshot taken here would read the state a previous call has set. """ - site_cache_state = self.site.get_cache_enabled() - self.site.enable_cache() if fetchSchemaParam is None: fetchSchemaParam = OSW._FetchSchemaParam() schema_title = fetchSchemaParam.schema_title @@ -1207,8 +1509,6 @@ def _fetch_schema( # reload the updated module, restoring the previous content if # the generated model turns out not to be importable reload_module_or_restore(model, result_model_path, previous_content) - if not site_cache_state: - self.site.disable_cache() # restore original state return OSW.FetchSchemaResult( fetched_schema_titles=fetchSchemaParam.fetched_schema_titles, @@ -1390,44 +1690,136 @@ def load_entity( entity = None schemas = [] schemas_fetched = True + # maps a category (page title, e.g. "Category:OSW...") to the + # class to use for it, either an already registered class (e.g. a + # packaged model class) or a freshly compiled one + category_to_cls: Dict[str, Type[model.Entity]] = {} jsondata = page.get_slot_content("jsondata") if param.remove_empty: remove_empty(jsondata) if jsondata: for category in jsondata["type"]: - schema = ( - self.site - .get_page( - WtSite.GetPageParam( - titles=[category], offline_pages=param.offline_pages + # one unusable category must not end the whole call + try: + schema = ( + self.site + .get_page( + WtSite.GetPageParam( + titles=[category], + offline_pages=param.offline_pages, + ) ) + .pages[0] + .get_slot_content("jsonschema") ) - .pages[0] - .get_slot_content("jsonschema") - ) - schemas.append(schema) - # generate model if not already exists - cls_name: str = schema["title"] - # If a schema_to_use is provided, we do not need to check if - # the model exists - if not param.model_to_use: - if not hasattr(model, cls_name): - if param.autofetch_schema: - self.fetch_schema( - OSW.FetchSchemaParam( - schema_title=category, - mode="append", - offline_pages=param.offline_pages, - ) + schemas.append(schema) + # generate model if not already exists + cls_name: str = schema["title"] + # If a schema_to_use is provided, we do not need to check if + # the model exists + if not param.model_to_use: + # Prefer a class already registered for this category + # IRI (e.g. a packaged model class) over compiling a + # new one. Compiling one anyway would take over the + # oold type registry entry for this category and hide + # the packaged class' typed fields/helpers. + registered_cls = oold_type_registry.get(category) + # Controllers and result wrappers (e.g. + # WikiFileController, UploadFileResult) inherit + # their category IRI from the model class they + # extend, and the registry keeps whichever class + # was defined last. Such a specialization asks for + # fields a plain page's jsondata does not carry, so + # prefer the canonical class from osw.model.entity + # over a subclass of it. + canonical_cls = getattr(model, cls_name, None) + if ( + registered_cls is not None + # the schema title can also name a module + # attribute that is not a class + and isinstance(canonical_cls, type) + and registered_cls is not canonical_cls + and issubclass(registered_cls, canonical_cls) + ): + _logger.debug( + f"Ignoring '{registered_cls}' registered for " + f"category '{category}': it specializes " + f"'{canonical_cls}', which is used instead." ) - if not hasattr(model, cls_name): - schemas_fetched = False - _logger.error( - f"Model {cls_name} not found. Schema {category} " - f"needs to be fetched first." - ) + registered_cls = None + if registered_cls is not None: + category_to_cls[category] = registered_cls + else: + if not hasattr(model, cls_name): + if param.autofetch_schema: + self.fetch_schema( + OSW.FetchSchemaParam( + schema_title=category, + mode="append", + offline_pages=param.offline_pages, + ) + ) + if not hasattr(model, cls_name): + schemas_fetched = False + _logger.error( + f"Model {cls_name} not found. Schema " + f"{category} needs to be fetched first." + ) + else: + generated_cls = getattr(model, cls_name) + # The class we are about to use may have just + # claimed (or may already hold) the registry + # slot for this category. If a different class + # is registered for it, someone's registration + # was silently overwritten - do not raise, but + # make sure this does not pass silently. + conflicting_cls = oold_type_registry.get( + category + ) + if ( + conflicting_cls is not None + and conflicting_cls is not generated_cls + # the slot holder specializes + # generated_cls, e.g. a controller + # that inherits the category IRI: + # the case the guard above expects, + # not a conflict. A fetch can leave + # a non-class under the schema title. + and not ( + isinstance(generated_cls, type) + and issubclass( + conflicting_cls, generated_cls + ) + ) + ): + _logger.warning( + f"Class '{generated_cls}' generated for " + f"category '{category}' claims the oold " + f"type registry slot already held by a " + f"different class '{conflicting_cls}'." + ) + category_to_cls[category] = generated_cls + except Exception as e: + schemas_fetched = False + _logger.error( + f"Error reading the schema of category " + f"{category} for page {page.title}: {e}" + ) + continue if not schemas_fetched: continue + # fetch_schema() reloads osw.model.entity, which replaces every + # class defined there, so a class picked before a later fetch in + # the loop above may be out of date: take the current object + for category, chosen_cls in category_to_cls.items(): + # a non-class is left for the construction below to reject + if ( + isinstance(chosen_cls, type) + and chosen_cls.__module__ == model.__name__ + ): + category_to_cls[category] = getattr( + model, chosen_cls.__name__, chosen_cls + ) try: if param.model_to_use: @@ -1437,13 +1829,15 @@ def load_entity( _logger.error("no schema defined") elif len(schemas) == 1: - cls: Type[model.Entity] = getattr(model, schemas[0]["title"]) + # category_to_cls is fully populated for every category + # that reached this point (see the loop above) + cls: Type[model.Entity] = category_to_cls[jsondata["type"][0]] entity: model.Entity = cls(**jsondata) else: bases = [] - for schema in schemas: - bases.append(getattr(model, schema["title"])) + for category in jsondata["type"]: + bases.append(category_to_cls[category]) cls = create_model("Test", __base__=tuple(bases)) entity: model.Entity = cls(**jsondata) except Exception as e: @@ -1570,6 +1964,19 @@ def _sync_per_property(self) -> None: f"'{self.overwrite.value}', which acts on the entity as a " f"whole. Use an OverwriteOptions value for 'overwrite'." ) + if self.per_property: + unknown = [ + key for key in self.per_property if key not in self.model.__fields__ + ] + if unknown: + # validate_per_property() only runs at construction and when + # 'per_property' itself is assigned. Reassigning 'model' would + # otherwise just drop the now-unknown keys from the rebuilt + # mapping, leaving a setting that silently never applies. + raise ValueError( + f"Property not found in model '{self.model.__name__}': " + f"{', '.join(unknown)}" + ) per_property_ = self.per_property or {} self._per_property = { field_name: per_property_.get(field_name, self.overwrite) @@ -1637,6 +2044,51 @@ def __init__(self, **data): overwrite=self.policy, ) + def _check_property_title_unchanged( + self, entity: model.Entity, entity_title: str + ) -> None: + """Raises if a property page with the uuid of 'entity' exists under another + title, since writing would create a second property page with that uuid. + + Parameters + ---------- + entity + The property entity about to be stored + entity_title + The full target page title, e.g. 'Property:HasName' + + Raises + ------ + ValueError + If the wiki holds the uuid on a differing 'Property:' page + """ + uuid = getattr(entity, "uuid", None) + if not uuid: + return + + def normalize(full_title: str) -> str: + # MediaWiki: '_' equals ' ' and the first character is case-insensitive + namespace, _sep, name = full_title.replace("_", " ").partition(":") + return f"{namespace}:{name[:1].upper()}{name[1:]}" + + # the base Entity @context maps uuid to the SMW property HasUuid + existing = self.site.semantic_search(f"[[HasUuid::{uuid!s}]]") + existing = [t for t in existing if t.startswith("Property:")] + differing = [t for t in existing if normalize(t) != normalize(entity_title)] + if not differing: + return + existing_str = ", ".join(f"'{t}'" for t in differing) + raise ValueError( + f"Cannot store property with uuid {uuid}: the wiki already holds it " + f"as {existing_str}, but the target title is '{entity_title}'. " + "Writing would create a second property page with the same uuid, " + "which happens when 'name' (or 'meta.wiki_page.title') changed. " + "If a replacement/rename of the property is intended, move the page " + f"'{differing[0]}' to '{entity_title}' in the wiki first, then repeat " + "the update. Otherwise keep the original 'name', or pass the existing " + "title in 'meta.wiki_page.title'." + ) + @staticmethod def _apply_overwrite_policy(param: OSW._ApplyOverwriteParam) -> WtPage: if param.inplace: @@ -1712,7 +2164,14 @@ def set_content(content_to_set: dict) -> None: # We want those not to be listed as keys local_content["jsondata"] = json.loads(param.entity.json(exclude_none=True)) if param.remove_empty: + # an empty value the caller assigned on purpose is how a property gets + # cleared, so it has to survive remove_empty() and reach the merge + # below, where step c) can let it win over the remote value + explicitly_set_empty = _explicitly_set_empty( + param.entity, local_content["jsondata"] + ) remove_empty(local_content["jsondata"]) + local_content["jsondata"].update(explicitly_set_empty) if param.debug: _logger.debug(f"'local_content': {local_content!s}") # Apply the overwrite logic @@ -1781,9 +2240,28 @@ class StoreEntityParam(OswBaseModel): overwrite_per_class: Optional[List[OSW.OverwriteClassParam]] = None """A list of OverwriteClassParam objects. If a class specific overwrite setting is set, this setting is used. + + Each object is copied when it is validated into this parameter, so it is a + snapshot taken here: configure a policy fully before passing it, since + mutating it afterwards no longer affects this StoreEntityParam. + + A class is identified by its OSW type. A subclass that does not redefine + 'type' therefore shares its parent's policy and cannot be given one of its + own. Generated model classes always redefine it; hand-written subclasses + such as the file controllers do not. """ remove_empty: Optional[bool] = True - """If true, remove key with an empty string value from the jsondata.""" + """If true, remove key with an empty string value from the jsondata. + + A top-level property that was explicitly assigned an empty value is kept + rather than stripped, so that '', [] or {} reaches the merge. Whether it + then clears the stored value still depends on the overwrite setting for + that property: 'true' clears it, 'only empty' clears it only if the stored + value is empty too, and 'false' (or 'keep existing', the default) leaves + the stored value in place. + + Assign None instead of an empty value for 'not known', otherwise the + property is cleared on the wiki.""" change_id: Optional[str] = None """ID to document the change. Entities within the same store_entity() call will share the same change_id. This parameter can also be used to link multiple @@ -1806,11 +2284,9 @@ class StoreEntityParam(OswBaseModel): seconds later if a page is reported as missing. If the query itself fails, the pages are reported as stored and an error is logged. Has no effect if 'offline' is True.""" - _overwrite_per_class: Dict[str, Dict[str, OSW.OverwriteClassParam]] = ( - PrivateAttr() - ) - """Private attribute, for internal use only. Use 'overwrite_per_class' - instead.""" + _overwrite_per_class: Dict[str, OSW.OverwriteClassParam] = PrivateAttr() + """Private attribute, for internal use only. Maps the OSW type of a model + class to its policy. Use 'overwrite_per_class' instead.""" def __init__(self, **data): super().__init__(**data) @@ -1836,22 +2312,32 @@ def __init__(self, **data): ) if self.overwrite is None: self.overwrite = self.__fields__["overwrite"].get_default() - self._overwrite_per_class = {"by name": {}, "by type": {}} + self._overwrite_per_class = {} if self.overwrite_per_class is not None: for param in self.overwrite_per_class: - model_name = param.model.__name__ model_type = param.model.__fields__["type"].get_default()[0] - if ( - model_name in self._overwrite_per_class["by name"].keys() - or model_type in self._overwrite_per_class["by type"].keys() - ): + declared = self._overwrite_per_class.get(model_type) + if declared is not None: + other = declared.model + detail = "" + # issubclass(X, X) is True, so the same class declared + # twice would otherwise be blamed on inheritance + if param.model is not other and ( + issubclass(param.model, other) + or issubclass(other, param.model) + ): + detail = ( + " A subclass that does not redefine 'type' inherits " + "it, so store_entity() cannot tell the two apart. " + "Declare a single OverwriteClassParam for both." + ) raise ValueError( - f"More than one OverwriteClassParam for the class " - f"'{model_type}' ({model_name}) has been passed in the " - f"list to 'overwrite_per_class'!" + f"More than one OverwriteClassParam for the type " + f"'{model_type}' has been passed in the list to " + f"'overwrite_per_class': '{other.__name__}' and " + f"'{param.model.__name__}'.{detail}" ) - self._overwrite_per_class["by name"][model_name] = param - self._overwrite_per_class["by type"][model_type] = param + self._overwrite_per_class[model_type] = param class StoreEntityResult(OswBaseModel): """Result of store_entity()""" @@ -1859,8 +2345,13 @@ class StoreEntityResult(OswBaseModel): change_id: str """The ID of the change""" pages: Dict[str, WtPage] - """The pages that have been successfully stored, keyed by full page title. - On partial failure this contains only the successfully-stored pages.""" + """The pages that have been successfully processed, keyed by full page title. + On partial failure this contains only the successfully-processed pages. + Includes the pages listed in 'skipped', which were left untouched.""" + skipped: Dict[str, WtPage] = {} + """Existing pages that were not written because the applicable overwrite + setting was 'keep existing', keyed by full page title. These also appear in + 'pages'; subtract them to get the pages that were actually written.""" failed: Dict[str, Exception] = {} """Entities that could not be stored, mapped to the exception that caused the failure. Empty on full success. The key is the full page title where one could @@ -1875,13 +2366,20 @@ class StoreEntityPartialError(Exception): """Raised by store_entity() when one or more entities could not be stored. Carries the partial ``StoreEntityResult`` so callers can learn exactly which - entities were written (``stored`` / ``result.pages``) and which failed - (``failed`` / ``result.failed``) without a separate existence query. + entities were written (``stored``), which existed already and were left + untouched by 'keep existing' (``skipped``), and which failed (``failed`` / + ``result.failed``) without a separate existence query. """ def __init__(self, result: OSW.StoreEntityResult): self.result = result - self.stored = list(result.pages.keys()) + # a page kept by 'keep existing' was not written, so it does not + # belong in 'stored'. 'result.pages' still lists it, for callers + # that only care about which entities are on the wiki afterwards. + self.skipped = list(result.skipped.keys()) + self.stored = [ + title for title in result.pages if title not in result.skipped + ] self.failed = result.failed total = len(result.pages) + len(result.failed) failed_titles = ", ".join(result.failed.keys()) @@ -1979,37 +2477,39 @@ def store_entity( max_index = len(param.entities) created_pages = {} + skipped_pages = {} edited_titles = set() """Titles of the pages an edit was sent for, to be verified below.""" meta_category_templates = {} + definitions_section = False if param.namespace == "Category": meta_category_titles = param.meta_category_title if not isinstance(meta_category_titles, list): meta_category_titles = [meta_category_titles] - meta_category_template_strs = {} - # We have to do this iteratively to support meta categories inheritance - while meta_category_titles is not None and len(meta_category_titles) > 0: - meta_categories = self.site.get_page( - WtSite.GetPageParam(titles=meta_category_titles) - ).pages - for meta_category in meta_categories: - meta_category_template_strs[meta_category.title] = ( - meta_category.get_slot_content("schema_template") - ) - meta_category_titles = meta_category.get_slot_content("jsondata").get( - "subclass_of" - ) + def get_pages(titles): + return self.site.get_page(WtSite.GetPageParam(titles=titles)).pages - for title in meta_category_template_strs.keys(): - meta_category_template_str = meta_category_template_strs[title] - if meta_category_template_str: - meta_category_templates[title] = compile_handlebars_template( - meta_category_template_str + # The templates follow the allOf chain of the schema of the meta + # categories, like the form editor does. subclass_of is not used: + # Category:Category has no subclass_of but an allOf to CharacteristicType + template_titles = collect_category_template_titles( + get_pages, meta_category_titles + ) + for page in get_pages(template_titles): + template_str = page.get_slot_content("schema_template") + if template_str: + meta_category_templates[page.title] = compile_handlebars_template( + template_str ) - # inverse order to have the most generic template first - meta_category_templates = dict(reversed(meta_category_templates.items())) + definitions_section = CHARACTERISTIC_TYPE_TITLE in template_titles + # keep the order of the list, the most generic template first + meta_category_templates = { + t: meta_category_templates[t] + for t in template_titles + if t in meta_category_templates + } def store_entity_( entity_: model.Entity, @@ -2043,6 +2543,8 @@ def store_entity_( if overwrite_class_param is None: raise TypeError("'overwrite_class_param' must not be None!") entity_title = namespace_ + ":" + title_ + if namespace_ == "Property" and param.offline is not True: + self._check_property_title_unchanged(entity_, entity_title) page = self._apply_overwrite_policy( OSW._ApplyOverwriteParam( page=WtPage( @@ -2091,28 +2593,43 @@ def store_entity_( ) mode = AggregateGeneratedSchemasParamMode.ROOT_LEVEL - # Put generated schema in definitions section, - # currently only enabled for Characteristics. The class is - # either the locally generated one or the one shipped by the - # opensemantic packages, depending on where the model is from. - characteristic_types = tuple( - t - for t in ( - getattr(model, "CharacteristicType", None), - _OpensemanticCharacteristicType, - ) - if t is not None - ) - if characteristic_types and isinstance(entity_, characteristic_types): + # Put the generated schema in the definitions section when the + # templates include CharacteristicType, like the form editor does. + if definitions_section: mode = AggregateGeneratedSchemasParamMode.DEFINITIONS_SECTION + definitions = ( + mode == AggregateGeneratedSchemasParamMode.DEFINITIONS_SECTION + ) + stored_schema = page.get_slot_content("jsonschema") or {} + # Regenerate the generated section from scratch. Skip this when + # the generation failed, so the stored section is kept. + if definitions and len(generated_schemas) > 0: + reset_generated_jsonld_context( + stored_schema, + isinstance(stored_schema.get("$defs"), dict) + and "generated" in stored_schema["$defs"], + ) + remove_generated_schema_section(stored_schema) + if definitions: + for generated_schema in generated_schemas.values(): + if "@context" in generated_schema: + merge_generated_jsonld_context( + stored_schema, generated_schema.pop("@context") + ) new_schema = aggregate_generated_schemas( AggregateGeneratedSchemasParam( - schema=page.get_slot_content("jsonschema"), + target_schema=stored_schema, generated_schemas=generated_schemas, mode=mode, ) ).aggregated_schema + if definitions and isinstance( + (new_schema.get("$defs") or {}).get("generated"), dict + ): + strip_root_members_superseded_by_generated( + new_schema, new_schema["$defs"]["generated"] + ) page.set_slot_content("jsonschema", new_schema) if param.offline is False and not kept_existing: page.edit( @@ -2128,6 +2645,8 @@ def store_entity_( f"'{page.get_url()}'." ) created_pages[page.title] = page + if kept_existing: + skipped_pages[page.title] = page sorted_entities = OSW.sort_list_of_entities_by_class(param.entities) _logger.info( @@ -2150,7 +2669,7 @@ class UploadObject(BaseModel): upload_index = 0 for class_type, entities in sorted_entities.by_type.items(): # Try to get a class specific overwrite setting - class_param = param._overwrite_per_class["by type"].get(class_type, None) + class_param = param._overwrite_per_class.get(class_type, None) if class_param is None: entity_model = entities[0].__class__ class_param = OSW.OverwriteClassParam( @@ -2253,7 +2772,10 @@ def failure_title_(upload_object: UploadObject) -> str: del created_pages[title] store_result = OSW.StoreEntityResult( - change_id=param.change_id, pages=created_pages, failed=failed + change_id=param.change_id, + pages=created_pages, + skipped=skipped_pages, + failed=failed, ) if failed: # Surface partial/total failure so callers cannot mistake a dropped @@ -2297,18 +2819,14 @@ def delete_entity_(entity_, comment_: str = None): comment_: Command for the change log, by default None """ - title_ = None namespace_ = None if hasattr(entity_, "meta"): if entity_.meta and entity_.meta.wiki_page: - if entity_.meta.wiki_page.title: - title_ = entity_.meta.wiki_page.title if entity_.meta.wiki_page.namespace: namespace_ = entity_.meta.wiki_page.namespace if namespace_ is None: namespace_ = get_namespace(entity_) - if title_ is None: - title_ = OSW.get_osw_id(entity_.uuid) + title_ = get_title(entity_) if namespace_ is None or title_ is None: _logger.error("Unsupported entity type") return diff --git a/src/osw/express.py b/src/osw/express.py index 50da2d03..d9525fbc 100644 --- a/src/osw/express.py +++ b/src/osw/express.py @@ -54,7 +54,11 @@ class OswExpress(OSW): """This class provides convenience functions for osw-python.""" domain: str - """The domain of the OSL instance to connect to.""" + """The domain of the OSL instance to connect to, as a bare host with an + optional port, e.g. 'wiki.example.org' or 'localhost:8081'.""" + scheme: str = "https" + """The URL scheme used to connect, 'https' unless the domain was given with + an 'http://' prefix, e.g. for a local test wiki.""" cred_filepath: Optional[Union[str, Path]] """The filepath to the credentials file. Will be overwritten by the cred_filepath of cred_mngr, if it posses a non-None value. If cred_filepath is None, @@ -67,10 +71,18 @@ class Config: @validator("domain") def validate_domain(cls, v): - pattern = r"^(?!-)[A-Za-z0-9.-]{1,63}(?<!-)\.[A-Za-z]{2,}$" + # Dot-separated labels (a single label such as 'localhost' is allowed), + # each without a leading or trailing hyphen, and an optional port. + label = r"(?!-)[A-Za-z0-9-]{1,63}(?<!-)" + pattern = rf"^{label}(?:\.{label})*(?::\d{{1,5}})?$" assert re.match(pattern, v), "The domain is not valid." # noqa: S101 pydantic validator idiom return v + @validator("scheme") + def validate_scheme(cls, v): + assert v in ("http", "https"), "The scheme must be 'http' or 'https'." # noqa: S101 pydantic validator idiom + return v + @overload def __init__( self, @@ -111,6 +123,13 @@ def __init__( "\nIf no domain was set as environmental variable, 'domain' is a " "required argument." ) + # An 'http://' or 'https://' prefix selects the scheme; the domain itself + # is stored bare, because other code builds URLs and IRIs from it. + scheme = "https" + match = re.match(r"^(https?)://", domain, flags=re.IGNORECASE) + if match: + scheme = match.group(1).lower() + domain = domain[match.end() :].rstrip("/") if cred_mngr is not None and cred_mngr.cred_filepath is not None: # If a credential manager is explicitly defined, that should have priority cred_filepath = cred_mngr.cred_filepath[0] @@ -144,7 +163,7 @@ def __init__( cred_mngr = CredentialManager(cred_filepath=cred_filepath) # Test if domain is reachable try: - url = f"https://{domain}/wiki/Main_Page" + url = f"{scheme}://{domain}/wiki/Main_Page" response = requests.get(url) # noqa: S113 reachability probe, TODO add timeout if response.status_code == 200: # Domain is reachable @@ -155,8 +174,10 @@ def __init__( ) except Exception as e: raise ConnectionError(f"Could not connect to '{domain}'. Error: {e}") - site = WtSite(WtSite.WtSiteConfig(iri=domain, cred_mngr=cred_mngr)) - super().__init__(**{"site": site, "domain": domain}) + # WtSite defaults to https; only an http iri needs the scheme spelled out. + iri = domain if scheme == "https" else f"{scheme}://{domain}" + site = WtSite(WtSite.WtSiteConfig(iri=iri, cred_mngr=cred_mngr)) + super().__init__(**{"site": site, "domain": domain, "scheme": scheme}) self.cred_mngr = cred_mngr self.cred_filepath = cred_filepath _logger.info(f"Connected to '{domain}'.") diff --git a/src/osw/service/config.py b/src/osw/service/config.py index d8e4499f..687a9941 100644 --- a/src/osw/service/config.py +++ b/src/osw/service/config.py @@ -922,6 +922,19 @@ def derive_domain(iri: str) -> str: return _derive_domain(iri) +def derive_connect_domain(iri: str) -> str: + """Return the domain argument for ``OswExpress`` that connects to ``iri``. + + This is the bare domain, prefixed with ``http://`` when ``iri`` names that + scheme, so that an http-only wiki (e.g. a local test wiki) is not reached + over https. + """ + domain = _derive_domain(iri) + if iri.lower().startswith("http://"): + return f"http://{domain}" + return domain + + def set_active_instance(iri: str) -> None: """Set the active instance to ``iri``. diff --git a/src/osw/service/context.py b/src/osw/service/context.py index eb405bf9..3127e73a 100644 --- a/src/osw/service/context.py +++ b/src/osw/service/context.py @@ -108,7 +108,9 @@ def osw(self) -> OswExpress: ``CredentialManager`` and passed to ``OswExpress`` explicitly. """ if self._osw is None: - self._osw = self.osw_for(self._require_active_domain()) + self._require_active_domain() + iri = config.get_active_iri() + self._osw = self.osw_for(config.derive_connect_domain(iri)) return self._osw @osw.setter diff --git a/src/osw/service/errors.py b/src/osw/service/errors.py index 3c8d5a16..5b32ce0e 100644 --- a/src/osw/service/errors.py +++ b/src/osw/service/errors.py @@ -127,3 +127,14 @@ class BinaryContent(OpError): type = "BinaryContent" exit_code = 3 + + +class StoreFailed(OpError): + """``store_entity`` could not store an entity. + + Keeps the wire ``type`` a non-``OpError`` ``StoreEntityPartialError`` had, + while the message carries the reason of each failed page. + """ + + type = "StoreEntityPartialError" + exit_code = 1 diff --git a/src/osw/service/ops/entities.py b/src/osw/service/ops/entities.py index f34c71de..cb8597ca 100644 --- a/src/osw/service/ops/entities.py +++ b/src/osw/service/ops/entities.py @@ -134,6 +134,10 @@ def create_or_update_entity( and writes a second entity instead of changing the intended one. Read the current record with ``get_entity`` first and keep its uuid. + The page name of an entity of a ``Category:Property`` subclass is + ``Property:<name>``. An update that changes the ``name`` of a property is + refused: move the page in the wiki first. + ``overwrite`` controls update behavior: one of true | false | only empty | replace remote | keep existing. The default, true, merges field by field into the stored record: fields absent from ``jsondata`` keep their stored @@ -163,15 +167,21 @@ def create_or_update_entity( raise errors.ValidationError( f"jsondata does not validate against {category}: {exc}" ) - store = ctx.osw.store_entity( - OSW.StoreEntityParam( - entities=[entity], - namespace=namespace, - overwrite=_parse_overwrite(overwrite), - edit_comment=comment, - bot_edit=True, + try: + store = ctx.osw.store_entity( + OSW.StoreEntityParam( + entities=[entity], + namespace=namespace, + overwrite=_parse_overwrite(overwrite), + edit_comment=comment, + bot_edit=True, + ) ) - ) + except OSW.StoreEntityPartialError as exc: + # the exception text lists titles only, the reasons are in .failed + raise errors.StoreFailed( + "; ".join(f"{title}: {reason}" for title, reason in exc.failed.items()) + ) from exc titles = list(store.pages.keys()) # WtPage.exists is set once, when the page is loaded, and no write path # refreshes it, so after the store it still reports the pre-write state. diff --git a/src/osw/tools/__init__.py b/src/osw/tools/__init__.py new file mode 100644 index 00000000..406eb4bd --- /dev/null +++ b/src/osw/tools/__init__.py @@ -0,0 +1 @@ +"""Maintenance and automation tools built on top of the osw core library.""" diff --git a/src/osw/tools/user_sync/__init__.py b/src/osw/tools/user_sync/__init__.py new file mode 100644 index 00000000..606890f8 --- /dev/null +++ b/src/osw/tools/user_sync/__init__.py @@ -0,0 +1,22 @@ +"""Sync OSW User items from MediaWiki accounts and public ORCID data.""" + +from __future__ import annotations + +from .config import ( + ORGANIZATION_CATEGORY, + USER_CATEGORY, + SyncConfig, + build_arg_parser, + config_from_args, +) +from .sync import SyncReport, run_user_sync + +__all__ = [ + "ORGANIZATION_CATEGORY", + "USER_CATEGORY", + "SyncConfig", + "SyncReport", + "build_arg_parser", + "config_from_args", + "run_user_sync", +] diff --git a/src/osw/tools/user_sync/build.py b/src/osw/tools/user_sync/build.py new file mode 100644 index 00000000..5ba89430 --- /dev/null +++ b/src/osw/tools/user_sync/build.py @@ -0,0 +1,91 @@ +"""Build pydantic User/Organization entities from proposals. + +Must run within an open OSW connection (the models resolve linked entities on +construction). Organization references are passed as a list of page-title IRIs; +the reference attribute is never read back here, as that would trigger a +network resolve. + +An empty field is set to None, so its key is absent from the serialized item. +The store merges with the stored page, so an absent key keeps the stored value. +Clearing a stored value is done by ``prune.clear_jsondata_keys`` instead. +""" + +from __future__ import annotations + +from typing import Any, List, Set + +from opensemantic.base.v1 import Organization, User + +from .config import PROTECTED_FIELDS +from .mapping import ProposedOrganization, ProposedUser + +# The concrete DisplayName type used by the label field. +_DISPLAY_NAME = User.__fields__["label"].type_ + + +def _labels(text: str) -> List[Any]: + return [_DISPLAY_NAME(text=text)] + + +def _strip_protected(entity: Any) -> None: + """Unset protected relations (e.g. employment_contract_status). + + The attribute is set to None, so the key is absent from the serialized item + and the model default is not written. A stored value is left unchanged. + """ + for name in PROTECTED_FIELDS: + setattr(entity, name, None) + + +def build_organization(proposed: ProposedOrganization) -> Organization: + return Organization(uuid=proposed.uuid, label=_labels(proposed.name)) + + +def build_user(proposed: ProposedUser) -> User: + """Build a full User entity for creation (never carries protected fields).""" + args: dict = { + "uuid": proposed.uuid, + "username": proposed.username, + "first_name": proposed.first_name, + "surname": proposed.surname, + "label": _labels(proposed.label), + } + if proposed.orcid: + args["orcid"] = proposed.orcid + if proposed.emails: + args["email"] = set(proposed.emails) + if proposed.websites: + args["website"] = set(proposed.websites) + if proposed.organizations: + args["organization"] = list(proposed.organizations) + entity = User(**args) + _strip_protected(entity) + return entity + + +def apply_update(entity: Any, proposed: ProposedUser, apply_fields: Set[str]) -> Any: + """Apply the accepted fields onto a loaded existing entity, in place. + + An empty organization or website value is set to None, which leaves the + stored value unchanged. Protected fields are always unset. + """ + if "username" in apply_fields: + entity.username = proposed.username + if "label" in apply_fields: + entity.label = _labels(proposed.label) + if "first_name" in apply_fields: + entity.first_name = proposed.first_name + if "surname" in apply_fields: + entity.surname = proposed.surname + if "orcid" in apply_fields: + entity.orcid = proposed.orcid + if "emails" in apply_fields: + entity.email = set(proposed.emails) + if "websites" in apply_fields: + entity.website = set(proposed.websites) if proposed.websites else None + if "organizations" in apply_fields: + entity.organization = ( + list(proposed.organizations) if proposed.organizations else None + ) + _strip_protected(entity) + return entity diff --git a/src/osw/tools/user_sync/config.py b/src/osw/tools/user_sync/config.py new file mode 100644 index 00000000..8d0bac7e --- /dev/null +++ b/src/osw/tools/user_sync/config.py @@ -0,0 +1,165 @@ +"""Configuration for the user-item sync tool.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass, field +from typing import List, Optional, Set + +# Category page titles of the target item types (see opensemantic.base.v1). +USER_CATEGORY = "Category:OSWd9aa0bca9b0040d8af6f5c091bf9eec7" +ORGANIZATION_CATEGORY = "Category:OSW1969007d5acf40539642877659a02c23" + +# Public ORCID API used to enrich ORCID users. +ORCID_API_BASE_DEFAULT = "https://pub.orcid.org/v3.0" + +# MediaWiki groups whose members are treated as bot/system accounts and skipped. +# All are standard MediaWiki groups, so this is not a per-instance skip-list. +SYSTEM_GROUPS = ["bot", "sysop", "bureaucrat", "interface-admin"] + +# User fields the script must never write and must remove from existing items +# (data protection). +PROTECTED_FIELDS = ("employment_contract_status",) + + +@dataclass +class SyncConfig: + """Runtime options for a single sync run.""" + + domain: Optional[str] = None + cred_filepath: Optional[str] = None + dry_run: bool = False + auto_apply: bool = False + limit: Optional[int] = None + include_orcid: bool = True + include_non_orcid: bool = True + exclude_bot_group: bool = True + exclude_system_usernames: bool = True + create_redirects: bool = True + # Optional enrichment beyond core identity, opt-in (off by default). + # Email is not optional: it is always attempted, and missing emails are + # reported as warnings rather than failing the run. + include_websites: bool = False + link_organizations: bool = False + # Remove disabled optional fields and protected fields from existing items. + prune: bool = False + # Blocked MediaWiki accounts are skipped unless this is set. + include_blocked: bool = False + orcid_api_base: str = ORCID_API_BASE_DEFAULT + excluded_groups: List[str] = field(default_factory=lambda: list(SYSTEM_GROUPS)) + + def enabled_optional_fields(self) -> Set[str]: + """Reconcile field names for the enabled optional data.""" + enabled: Set[str] = set() + if self.include_websites: + enabled.add("websites") + if self.link_organizations: + enabled.add("organizations") + return enabled + + +def build_arg_parser() -> argparse.ArgumentParser: + """Build the argument parser used by the example wrapper.""" + parser = argparse.ArgumentParser( + description="Create or update OSW User items from MediaWiki and ORCID.", + ) + parser.add_argument("--domain", help="Target OSL domain, e.g. llm4eln.semos.dev.") + parser.add_argument( + "--cred-filepath", + dest="cred_filepath", + help="Path to accounts.pwd.yaml (defaults to the working directory).", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Show the preview only; do not write anything.", + ) + parser.add_argument( + "--auto-apply", + dest="auto_apply", + action="store_true", + help="Non-interactive: apply creates, gap-fills and removals; keep existing on conflicts.", + ) + parser.add_argument( + "--limit", + type=int, + help="Process at most this many MediaWiki users (for testing).", + ) + parser.add_argument( + "--no-redirects", + dest="create_redirects", + action="store_false", + help="Do not create User: redirect pages.", + ) + parser.add_argument( + "--with-websites", + dest="include_websites", + action="store_true", + help="Also store ORCID researcher URLs on the user item (opt-in).", + ) + parser.add_argument( + "--with-organizations", + dest="link_organizations", + action="store_true", + help="Also resolve ORCID affiliations to Organization items and link them.", + ) + parser.add_argument( + "--with-extras", + dest="with_extras", + action="store_true", + help="Enable all optional data: websites and organizations.", + ) + parser.add_argument( + "--prune", + dest="prune", + action="store_true", + help="Overwrite disabled optional fields of existing items with an empty " + "list and remove employment_contract_status (data cleanup). Off by " + "default; a normal run only adds.", + ) + account_scope = parser.add_mutually_exclusive_group() + account_scope.add_argument( + "--orcid-only", + dest="include_non_orcid", + action="store_false", + help="Sync only accounts whose username is an ORCID iD.", + ) + account_scope.add_argument( + "--mw-only", + dest="include_orcid", + action="store_false", + help="Sync only non-ORCID (MediaWiki-native) accounts.", + ) + parser.add_argument( + "--include-system", + dest="exclude_system_usernames", + action="store_false", + help="Do not skip MediaWiki reserved system accounts (Maintenance script etc.).", + ) + parser.add_argument( + "--include-blocked", + dest="include_blocked", + action="store_true", + help="Do not skip blocked MediaWiki accounts.", + ) + return parser + + +def config_from_args(argv: Optional[List[str]] = None) -> SyncConfig: + """Parse command-line arguments into a SyncConfig.""" + args = build_arg_parser().parse_args(argv) + return SyncConfig( + domain=args.domain, + cred_filepath=args.cred_filepath, + dry_run=args.dry_run, + auto_apply=args.auto_apply, + limit=args.limit, + include_orcid=args.include_orcid, + include_non_orcid=args.include_non_orcid, + exclude_system_usernames=args.exclude_system_usernames, + create_redirects=args.create_redirects, + include_websites=args.include_websites or args.with_extras, + link_organizations=args.link_organizations or args.with_extras, + prune=args.prune, + include_blocked=args.include_blocked, + ) diff --git a/src/osw/tools/user_sync/existing.py b/src/osw/tools/user_sync/existing.py new file mode 100644 index 00000000..5ba79801 --- /dev/null +++ b/src/osw/tools/user_sync/existing.py @@ -0,0 +1,167 @@ +"""Load existing OSW User items so the sync can update in place. + +Matches are keyed on the ``username`` field, so accounts already stored under a +different uuid are updated rather than duplicated. +""" + +from __future__ import annotations + +import copy +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Tuple + +from osw.core import OSW +from osw.wiki_tools import SearchParam +from osw.wtsite import WtSite + +from .config import USER_CATEGORY + +# Results per semantic search request. +PAGE_SIZE = 1000 + + +@dataclass +class ExistingUsers: + """Existing User items indexed by their MediaWiki username.""" + + by_username: Dict[str, Any] = field(default_factory=dict) + # The jsondata as stored on the page, per username. The loaded entity cannot + # tell a stored value from a model default. + stored_jsondata: Dict[str, dict] = field(default_factory=dict) + + def get(self, username: str) -> Optional[Any]: + return self.by_username.get(username) + + def get_stored(self, username: str) -> Optional[dict]: + return self.stored_jsondata.get(username) + + +def _as_entity_list(result: Any) -> List[Any]: + """Normalize load_entity output (single, list, or LoadEntityResult).""" + if result is None: + return [] + entities = getattr(result, "entities", None) + if entities is not None: + return list(entities) + if isinstance(result, list): + return result + return [result] + + +def _load_with_stored(osw: Any, titles: List[str]) -> Tuple[List[Any], Dict[str, dict]]: + """Load the entities and read the stored jsondata of the same pages. + + The pages are fetched once and handed to load_entity as offline pages, so the + jsondata is read without a second request. A copy is taken first, because + load_entity removes empty keys from the slot content in place. + + Returns: + The entities and the stored jsondata keyed by the uuid in it. + """ + pages = osw.site.get_page(WtSite.GetPageParam(titles=titles)).pages + offline_pages = {page.title: page for page in pages} + stored: Dict[str, dict] = {} + for page in pages: + jsondata = page.get_slot_content("jsondata") + if isinstance(jsondata, dict): + stored[str(jsondata.get("uuid"))] = copy.deepcopy(jsondata) + result = osw.load_entity( + OSW.LoadEntityParam(titles=titles, offline_pages=offline_pages) + ) + return _as_entity_list(result), stored + + +def _query_user_titles( + osw: Any, category: str, limit: Optional[int] = None +) -> List[str]: + """Return the titles of all items of a category, paging through the results. + + The next offset is the one the wiki reports (``query-continue-offset``), not + the offset plus the page size, because the wiki can cut a page shorter. + """ + titles: List[str] = [] + offset = 0 + while True: + query = SearchParam( + query=f"[[HasType::{category}]]|offset={offset}", + limit=PAGE_SIZE, + return_meta=True, + ) + result = osw.site.semantic_search(query)[0] + titles.extend(result.titles) + if limit is not None and len(titles) >= limit: + return titles[:limit] + if not result.truncated: + return titles + if result.next_offset is None or result.next_offset <= offset: + raise RuntimeError( + f"The wiki reported the next offset {result.next_offset} after " + f"offset {offset} for '{category}'. Paging cannot continue." + ) + offset = result.next_offset + + +def load_existing_users( + osw: Any, + category: str = USER_CATEGORY, + limit: Optional[int] = None, +) -> ExistingUsers: + """Query and load all existing User items, indexed by username. + + Args: + osw: An authenticated OSW connection exposing ``site.semantic_search`` + and ``load_entity``. + category: The User category page title. + limit: Load at most this many items (for testing). + + Raises: + RuntimeError: If the wiki reports a next offset that does not advance. + """ + titles = _query_user_titles(osw, category, limit) + if not titles: + return ExistingUsers() + entities, stored = _load_with_stored(osw, titles) + existing = ExistingUsers() + for entity in entities: + username = getattr(entity, "username", None) + if username: + existing.by_username[username] = entity + jsondata = stored.get(str(getattr(entity, "uuid", None))) + if jsondata is not None: + existing.stored_jsondata[username] = jsondata + return existing + + +def adopt_existing_pages( + osw: Any, existing: ExistingUsers, proposed_users: List[Any] +) -> List[str]: + """Add items that exist at a proposed user's deterministic page title. + + A user without a match by ``username`` would be created at its deterministic + page title. If that page already exists (for example, it has no username or + is not typed as a User), it is loaded and indexed under the proposed + username, so the sync reconciles against it instead of creating it again. + + Returns: + The page titles that were adopted. + """ + candidates = [p for p in proposed_users if existing.get(p.username) is None] + if not candidates: + return [] + titles = [p.full_page_title for p in candidates] + missing = set(osw._get_missing_page_titles(titles, confirm_delay_s=0)) + present = [p for p in candidates if p.full_page_title not in missing] + if not present: + return [] + entities, stored = _load_with_stored(osw, [p.full_page_title for p in present]) + by_uuid = {str(getattr(e, "uuid", None)): e for e in entities if e is not None} + adopted: List[str] = [] + for proposed in present: + entity = by_uuid.get(str(proposed.uuid)) + if entity is not None: + existing.by_username[proposed.username] = entity + jsondata = stored.get(str(proposed.uuid)) + if jsondata is not None: + existing.stored_jsondata[proposed.username] = jsondata + adopted.append(proposed.full_page_title) + return adopted diff --git a/src/osw/tools/user_sync/interactive.py b/src/osw/tools/user_sync/interactive.py new file mode 100644 index 00000000..835245ca --- /dev/null +++ b/src/osw/tools/user_sync/interactive.py @@ -0,0 +1,203 @@ +"""Interactive preview and conflict resolution for the sync. + +Rendering is pure (returns strings). All terminal IO goes through ``Prompter``, +which wraps ``input`` / ``print`` so tests can script answers and capture output. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Callable, List, Optional, Sequence, Set + +from .mapping import ProposedUser +from .reconcile import ( + CONFLICT, + GAP_FILL, + NEW, + RECONCILED_FIELDS, + REMOVE, + UNCHANGED, + ReconcilePlan, + UserChange, + proposed_fields, +) + +_MARKERS = {NEW: "+", GAP_FILL: "~", CONFLICT: "!", UNCHANGED: "="} + + +def _is_empty(value: Any) -> bool: + if value is None or value == "": + return True + return isinstance(value, (list, set, tuple)) and len(value) == 0 + + +def nonempty_fields(proposed: ProposedUser) -> Set[str]: + """Reconciled field names that carry a value on the proposed user.""" + pf = proposed_fields(proposed) + return {name for name in RECONCILED_FIELDS if not _is_empty(pf[name])} + + +def render_preview(plan: ReconcilePlan) -> List[str]: + """Human-readable preview: a header line plus one row per user.""" + counts = plan.counts() + lines = [ + f"Users: {len(plan.changes)} | " + f"new={counts[NEW]} gap_fill={counts[GAP_FILL]} " + f"conflict={counts[CONFLICT]} unchanged={counts[UNCHANGED]}" + ] + for change in plan.changes: + marker = _MARKERS[change.category] + detail = "" + if change.diffs: + names = [ + ("-" + d.name if d.status == REMOVE else d.name) for d in change.diffs + ] + detail = " (" + ", ".join(names) + ")" + placeholder = " [placeholder name]" if change.proposed.placeholder_name else "" + lines.append( + f" {marker} {change.proposed.username} {change.category}{detail}{placeholder}" + ) + return lines + + +def render_conflict(change: UserChange) -> List[str]: + """Field-by-field display of one user's differences.""" + lines = [f"User {change.proposed.username}:"] + for diff in change.diffs: + lines.append( + f" {diff.name} [{diff.status}]: " + f"existing={diff.existing!r} -> proposed={diff.proposed!r}" + ) + return lines + + +class Prompter: + """Thin IO seam over input/print for testable prompting.""" + + def __init__( + self, + input_fn: Callable[[str], str] = input, + output_fn: Callable[[str], None] = print, + ): + self._input = input_fn + self._output = output_fn + + def write(self, text: str = "") -> None: + self._output(text) + + def ask(self, prompt: str, choices: Sequence[str]) -> str: + keys = "/".join(choices) + while True: + answer = self._input(f"{prompt} [{keys}]: ").strip().lower() + if answer in choices: + return answer + self.write(f"Please choose one of: {keys}") + + def confirm(self, prompt: str, default: bool = False) -> bool: + suffix = "[Y/n]" if default else "[y/N]" + answer = self._input(f"{prompt} {suffix}: ").strip().lower() + if not answer: + return default + return answer in ("y", "yes") + + +@dataclass +class ResolvedChange: + """A user change with the decided action and fields to write.""" + + change: UserChange + action: str # "create" | "update" | "skip" + apply_fields: Set[str] = field(default_factory=set) + + +@dataclass +class Resolution: + """The outcome of resolving a plan: what to write and whether to proceed.""" + + resolved: List[ResolvedChange] = field(default_factory=list) + accepted: bool = False + + def creates(self) -> List[ResolvedChange]: + return [r for r in self.resolved if r.action == "create"] + + def updates(self) -> List[ResolvedChange]: + return [r for r in self.resolved if r.action == "update"] + + +def _review_user(change: UserChange, prompter: Prompter) -> Set[str]: + """Resolve one conflicting user's fields, returning accepted conflict fields.""" + for line in render_conflict(change): + prompter.write(line) + conflict_diffs = [d for d in change.diffs if d.status == CONFLICT] + choice = prompter.ask( + "Take all new / Keep all existing / Field-by-field?", ("a", "k", "f") + ) + if choice == "a": + return {d.name for d in conflict_diffs} + if choice == "k": + return set() + accepted: Set[str] = set() + for diff in conflict_diffs: + if prompter.ask(f"{diff.name}: keep or take new?", ("k", "n")) == "n": + accepted.add(diff.name) + return accepted + + +def resolve_plan( + plan: ReconcilePlan, + prompter: Prompter, + dry_run: bool = False, + auto_apply: bool = False, + summary_lines: Optional[Sequence[str]] = None, +) -> Resolution: + """Preview the plan and resolve conflicts into a write decision.""" + for line in render_preview(plan): + prompter.write(line) + for line in summary_lines or []: + prompter.write(line) + + conflict_mode: Optional[str] = None + if plan.conflicts and not dry_run: + if auto_apply: + conflict_mode = "keep" + else: + choice = prompter.ask( + "Conflicts found: Apply all / Keep existing / Review each / Abort?", + ("a", "k", "r", "x"), + ) + if choice == "x": + return Resolution(resolved=[], accepted=False) + conflict_mode = {"a": "all", "k": "keep", "r": "review"}[choice] + + resolved: List[ResolvedChange] = [] + for change in plan.changes: + if change.category == NEW: + resolved.append( + ResolvedChange(change, "create", nonempty_fields(change.proposed)) + ) + elif change.category == UNCHANGED: + resolved.append(ResolvedChange(change, "skip")) + elif change.category == GAP_FILL: + resolved.append( + ResolvedChange(change, "update", {d.name for d in change.diffs}) + ) + else: # CONFLICT + accepted = {d.name for d in change.diffs if d.status in (GAP_FILL, REMOVE)} + if conflict_mode == "all": + accepted |= {d.name for d in change.diffs if d.status == CONFLICT} + elif conflict_mode == "review": + accepted |= _review_user(change, prompter) + action = "update" if accepted else "skip" + resolved.append(ResolvedChange(change, action, accepted)) + + if dry_run: + accepted_run = False + elif auto_apply: + accepted_run = True + else: + n_create = sum(1 for r in resolved if r.action == "create") + n_update = sum(1 for r in resolved if r.action == "update") + accepted_run = prompter.confirm( + f"Proceed to create {n_create} and update {n_update} user items?" + ) + return Resolution(resolved=resolved, accepted=accepted_run) diff --git a/src/osw/tools/user_sync/mapping.py b/src/osw/tools/user_sync/mapping.py new file mode 100644 index 00000000..b5dcd414 --- /dev/null +++ b/src/osw/tools/user_sync/mapping.py @@ -0,0 +1,151 @@ +"""Map MediaWiki accounts and ORCID profiles to proposed OSW items. + +These functions are pure: they compute the field values and deterministic ids of +the items to store, without instantiating the pydantic models (which requires an +open OSW connection) or touching the network. The store step builds the actual +``User`` / ``Organization`` entities from these proposals. +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass, field +from typing import List, Optional, Tuple + +from osw.data.import_utility import uuid_to_full_page_title + +from .config import ORGANIZATION_CATEGORY, USER_CATEGORY +from .sources import MwUser, OrcidAffiliation, OrcidProfile + + +def _category_namespace(category: str) -> uuid.UUID: + """Derive the uuid5 namespace of a category from its page title.""" + return uuid.UUID(category.split("OSW")[-1]) + + +USER_NAMESPACE = _category_namespace(USER_CATEGORY) +ORGANIZATION_NAMESPACE = _category_namespace(ORGANIZATION_CATEGORY) + + +@dataclass +class ProposedOrganization: + """An Organization item derived from an ORCID affiliation.""" + + uuid: uuid.UUID + name: str + ror_id: Optional[str] = None + + @property + def full_page_title(self) -> str: + return uuid_to_full_page_title(self.uuid) + + +@dataclass +class ProposedUser: + """A User item derived from a MediaWiki account and optional ORCID data.""" + + uuid: uuid.UUID + username: str + first_name: str + surname: str + label: str + orcid: Optional[str] = None + emails: List[str] = field(default_factory=list) + websites: List[str] = field(default_factory=list) + organizations: List[str] = field(default_factory=list) + placeholder_name: bool = False + + @property + def full_page_title(self) -> str: + return uuid_to_full_page_title(self.uuid) + + +def normalize_org_name(name: str) -> str: + """Collapse whitespace and lowercase, for a stable org identity key.""" + return " ".join(name.lower().split()) + + +def user_uuid(mw_user: MwUser) -> uuid.UUID: + """Deterministic uuid5: keyed on ORCID iD for ORCID users, else username.""" + key = mw_user.orcid_id or mw_user.name + return uuid.uuid5(USER_NAMESPACE, key) + + +def organization_uuid(affiliation: OrcidAffiliation) -> uuid.UUID: + """Deterministic uuid5: keyed on ROR id when present, else the org name.""" + key = affiliation.ror_id or normalize_org_name(affiliation.organization) + return uuid.uuid5(ORGANIZATION_NAMESPACE, key) + + +def map_organization(affiliation: OrcidAffiliation) -> ProposedOrganization: + return ProposedOrganization( + uuid=organization_uuid(affiliation), + name=affiliation.organization, + ror_id=affiliation.ror_id, + ) + + +def _derive_names( + profile: Optional[OrcidProfile], username: str +) -> Tuple[str, str, str, bool]: + """Return (first_name, surname, label, is_placeholder). + + Prefers ORCID given/family names, then a split of the display name, then a + username-based placeholder when no name information is available. + """ + given = profile.given_names if profile else None + family = profile.family_name if profile else None + display = profile.display_name if profile else None + if given and family: + return given, family, display or f"{given} {family}", False + if display: + parts = display.split() + first = given or parts[0] + surname = family or (parts[-1] if len(parts) > 1 else parts[0]) + return first, surname, display, False + if given or family: + first = given or family + surname = family or given + return first, surname, f"{first} {surname}", False + return username, username, username, True + + +def map_user( + mw_user: MwUser, + profile: Optional[OrcidProfile] = None, + include_websites: bool = False, + link_organizations: bool = False, +) -> Tuple[ProposedUser, List[ProposedOrganization]]: + """Build a ProposedUser (and any linked organizations) from the sources. + + Email and names are always taken from the profile (core). Websites and + organizations are populated only when their flag is enabled. + """ + orcid_uri = f"https://orcid.org/{mw_user.orcid_id}" if mw_user.orcid_id else None + first, surname, label, placeholder = _derive_names(profile, mw_user.name) + + organizations: List[ProposedOrganization] = [] + org_titles: List[str] = [] + if link_organizations and profile: + seen = set() + for affiliation in profile.affiliations: + org = map_organization(affiliation) + if org.uuid in seen: + continue + seen.add(org.uuid) + organizations.append(org) + org_titles.append(org.full_page_title) + + proposed = ProposedUser( + uuid=user_uuid(mw_user), + username=mw_user.name, + first_name=first, + surname=surname, + label=label, + orcid=orcid_uri, + emails=list(profile.emails) if profile else [], + websites=list(profile.urls) if (profile and include_websites) else [], + organizations=org_titles, + placeholder_name=placeholder, + ) + return proposed, organizations diff --git a/src/osw/tools/user_sync/prune.py b/src/osw/tools/user_sync/prune.py new file mode 100644 index 00000000..f0dd85c7 --- /dev/null +++ b/src/osw/tools/user_sync/prune.py @@ -0,0 +1,67 @@ +"""Clear stored relations of existing user items (``--prune``). + +The store merges with the stored page, so an absent key keeps the stored value. +The keys are therefore changed in a second edit of the page's jsondata slot. +A key of a list or set field is overwritten with an empty list. A key of a +single-valued field is deleted, because an empty list is not a valid value for +it. A key that is not a field of User is deleted as well. +""" + +from __future__ import annotations + +import json +from typing import Any, Iterable + +from opensemantic.base.v1 import User +from pydantic.v1.fields import SHAPE_SINGLETON + +from osw.wtsite import WtPage + +_EMPTY = (None, [], "", {}) + + +def is_multi_valued(key: str) -> bool: + """True if the User field is a list or a set, False for any other key.""" + model_field = User.__fields__.get(key) + return model_field is not None and model_field.shape != SHAPE_SINGLETON + + +def needs_clearing(key: str, value: Any) -> bool: + """True if a present stored value of ``key`` must be cleared by pruning. + + A list or set field needs it only when it is not empty. Any other key needs + it whenever it is present, also with an empty value such as ``[]``, which is + not a valid value for a single-valued field. + """ + if is_multi_valued(key): + return value not in _EMPTY + return True + + +def clear_jsondata_keys(osw: Any, title: str, keys: Iterable[str]) -> bool: + """Clear each key in the page's jsondata slot that needs_clearing selects. + + A list or set field becomes ``[]``. Any other key is deleted. The page is + read with a new WtPage, which downloads the current revision and bypasses the + WtSite page cache. Other keys are written back unchanged. + Returns True if the page was written, False if nothing changed. + """ + page = WtPage(wtSite=osw.site, title=title) + content = page.get_slot_content("jsondata") + if isinstance(content, str): + content = json.loads(content) + if not isinstance(content, dict): + return False + changed = False + for key in keys: + if key in content and needs_clearing(key, content[key]): + if is_multi_valued(key): + content[key] = [] + else: + del content[key] + changed = True + if not changed: + return False + page.set_slot_content("jsondata", content) + page.edit(comment="user-sync: prune") + return True diff --git a/src/osw/tools/user_sync/reconcile.py b/src/osw/tools/user_sync/reconcile.py new file mode 100644 index 00000000..36f50ebf --- /dev/null +++ b/src/osw/tools/user_sync/reconcile.py @@ -0,0 +1,246 @@ +"""Reconcile proposed User items against the ones already stored. + +Pure logic: given proposed users and an index of existing items, classify each +user into one of four categories: NEW, GAP_FILL, CONFLICT or UNCHANGED. Each +changed field has one of three statuses: GAP_FILL, CONFLICT or REMOVE (REMOVE is +used for disabled optional fields and protected fields, only when pruning). A user +with only gap-fills or removals is GAP_FILL. Placeholder names are never compared +for an existing item. The interactive layer consumes this plan; nothing here does IO. +""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass, field +from typing import Any, Dict, FrozenSet, List, Optional + +from .config import PROTECTED_FIELDS +from .existing import ExistingUsers +from .mapping import ProposedUser +from .prune import needs_clearing + +NEW = "new" +GAP_FILL = "gap_fill" +CONFLICT = "conflict" +REMOVE = "remove" +UNCHANGED = "unchanged" + +# username is the match key and is never reconciled as a value. +SCALAR_FIELDS = ("label", "first_name", "surname", "orcid") +SET_FIELDS = ("emails", "websites", "organizations") +URL_FIELDS = ("orcid", "websites") +RECONCILED_FIELDS = SCALAR_FIELDS + SET_FIELDS +# Fields derived from a username placeholder when no name is known. +PLACEHOLDER_FIELDS = ("label", "first_name", "surname") +# Fields that are only written when opted in; removed from existing when disabled. +OPTIONAL_FIELDS = ("websites", "organizations") + + +def _is_empty(value: Any) -> bool: + if value is None or value == "": + return True + return isinstance(value, (list, set, tuple)) and len(value) == 0 + + +def _norm_url(value: Any) -> Optional[str]: + return None if value is None else str(value).rstrip("/") + + +def _norm_scalar(name: str, value: Any) -> Optional[str]: + if value is None: + return None + return _norm_url(value) if name in URL_FIELDS else str(value) + + +def _norm_set(name: str, value: Any) -> set: + if not value: + return set() + if name in URL_FIELDS: + return {_norm_url(x) for x in value} + return {str(x) for x in value} + + +def _existing_label(entity: Any) -> Optional[str]: + for label in getattr(entity, "label", None) or []: + text = getattr(label, "text", None) + if text: + return text + return None + + +def _existing_orgs(entity: Any) -> Any: + """Read organization page-title refs without resolving the relation. + + Loaded entities keep relation targets as IRIs in ``__iris__``; reading the + attribute itself would make oold resolve (and fail on) the linked items. + """ + iris = getattr(entity, "__iris__", None) + if isinstance(iris, dict): + value = iris.get("organization") + if value is None: + return None + return value if isinstance(value, (list, tuple, set)) else [value] + try: + return getattr(entity, "organization", None) + except Exception: # pragma: no cover - defensive + return None + + +def _existing_has_iri(entity: Any, name: str) -> bool: + """True if the loaded entity carries a relation IRI for ``name``.""" + iris = getattr(entity, "__iris__", None) + return isinstance(iris, dict) and bool(iris.get(name)) + + +def _has_protected_value( + entity: Any, name: str, stored_jsondata: Optional[Dict[str, Any]] +) -> bool: + """True if the protected field holds a value that pruning would remove. + + Without stored jsondata the loaded entity is the only source. + """ + if stored_jsondata is None: + return _existing_has_iri(entity, name) + if name not in stored_jsondata: + return False + return needs_clearing(name, stored_jsondata[name]) + + +def existing_fields(entity: Any) -> Dict[str, Any]: + """Normalized comparable field view of a loaded User entity.""" + return { + "label": _existing_label(entity), + "first_name": getattr(entity, "first_name", None), + "surname": getattr(entity, "surname", None), + "orcid": _norm_scalar("orcid", getattr(entity, "orcid", None)), + "emails": _norm_set("emails", getattr(entity, "email", None)), + "websites": _norm_set("websites", getattr(entity, "website", None)), + "organizations": _norm_set("organizations", _existing_orgs(entity)), + } + + +def proposed_fields(proposed: ProposedUser) -> Dict[str, Any]: + """Normalized comparable field view of a proposed user.""" + return { + "label": proposed.label, + "first_name": proposed.first_name, + "surname": proposed.surname, + "orcid": _norm_scalar("orcid", proposed.orcid), + "emails": _norm_set("emails", proposed.emails), + "websites": _norm_set("websites", proposed.websites), + "organizations": _norm_set("organizations", proposed.organizations), + } + + +@dataclass +class FieldDiff: + """A single field that would change on an existing item.""" + + name: str + existing: Any + proposed: Any + status: str # GAP_FILL, CONFLICT or REMOVE + + +@dataclass +class UserChange: + """The reconciliation outcome for one proposed user.""" + + proposed: ProposedUser + existing: Optional[Any] + category: str + diffs: List[FieldDiff] = field(default_factory=list) + + +def _compare(existing_val: Any, proposed_val: Any) -> Optional[str]: + """Per-field status, or None when there is nothing to propose.""" + if _is_empty(proposed_val): + return None + if _is_empty(existing_val): + return GAP_FILL + return UNCHANGED if existing_val == proposed_val else CONFLICT + + +def reconcile_user( + proposed: ProposedUser, + existing: Optional[Any], + enabled_fields: FrozenSet[str] = frozenset(), + prune: bool = False, + stored_jsondata: Optional[Dict[str, Any]] = None, +) -> UserChange: + if existing is None: + return UserChange(proposed=proposed, existing=None, category=NEW) + ef = existing_fields(existing) + pf = proposed_fields(proposed) + diffs: List[FieldDiff] = [] + has_conflict = has_change = False + for name in RECONCILED_FIELDS: + if proposed.placeholder_name and name in PLACEHOLDER_FIELDS: + # A placeholder is only written for a new item, never for an existing one. + continue + if name in OPTIONAL_FIELDS and name not in enabled_fields: + # Disabled optional field: remove any existing value only when pruning. + if prune and not _is_empty(ef[name]): + diffs.append(FieldDiff(name, ef[name], None, REMOVE)) + has_change = True + continue + status = _compare(ef[name], pf[name]) + if status in (GAP_FILL, CONFLICT): + diffs.append(FieldDiff(name, ef[name], pf[name], status)) + has_conflict = has_conflict or status == CONFLICT + has_change = has_change or status == GAP_FILL + # An item found by its page title instead of its username has no username + # yet. Fill it, so the next run matches the item by username. + if not getattr(existing, "username", None): + diffs.append(FieldDiff("username", None, proposed.username, GAP_FILL)) + has_change = True + # Protected fields are stripped from existing items only when pruning. The + # stored jsondata decides, because a loaded entity also shows model defaults. + if prune: + for name in PROTECTED_FIELDS: + if _has_protected_value(existing, name, stored_jsondata): + diffs.append(FieldDiff(name, "set", None, REMOVE)) + has_change = True + category = CONFLICT if has_conflict else GAP_FILL if has_change else UNCHANGED + return UserChange( + proposed=proposed, existing=existing, category=category, diffs=diffs + ) + + +@dataclass +class ReconcilePlan: + """All per-user reconciliation outcomes for a run.""" + + changes: List[UserChange] = field(default_factory=list) + + def by_category(self, category: str) -> List[UserChange]: + return [c for c in self.changes if c.category == category] + + @property + def conflicts(self) -> List[UserChange]: + return self.by_category(CONFLICT) + + def counts(self) -> Dict[str, int]: + counter = Counter(c.category for c in self.changes) + return { + cat: counter.get(cat, 0) for cat in (NEW, GAP_FILL, CONFLICT, UNCHANGED) + } + + +def reconcile( + proposed_users: List[ProposedUser], + existing: ExistingUsers, + enabled_fields: FrozenSet[str] = frozenset(), + prune: bool = False, +) -> ReconcilePlan: + changes = [ + reconcile_user( + p, + existing.get(p.username), + enabled_fields, + prune, + existing.get_stored(p.username), + ) + for p in proposed_users + ] + return ReconcilePlan(changes=changes) diff --git a/src/osw/tools/user_sync/redirects.py b/src/osw/tools/user_sync/redirects.py new file mode 100644 index 00000000..4f2cae79 --- /dev/null +++ b/src/osw/tools/user_sync/redirects.py @@ -0,0 +1,39 @@ +"""Create ``User:<username>`` redirect pages pointing at the user item. + +An existing user page with real content is never overwritten; only missing +pages or pages that are already redirects are (re)written. +""" + +from __future__ import annotations + +from typing import Any, Optional + +from osw.wtsite import WtPage + + +def redirect_wikitext(item_title: str) -> str: + return f"#REDIRECT [[{item_title}]]" + + +def is_redirect(content: str) -> bool: + return content.lstrip().upper().startswith("#REDIRECT") + + +def ensure_redirect(osw: Any, username: str, item_title: str) -> Optional[str]: + """Create or refresh ``User:<username>`` -> item redirect. + + Returns the page title if written, or None if skipped (a real page already + exists, or the redirect is already correct). + """ + title = f"User:{username}" + target = redirect_wikitext(item_title) + page = WtPage(wtSite=osw.site, title=title) + if getattr(page, "exists", False): + content = page.get_slot_content("main") or "" + if content.strip() and not is_redirect(content): + return None # do not clobber a real user page + if content.strip() == target: + return None # already correct + page.set_slot_content("main", target) + page.edit(comment="user-sync: redirect to user item") + return title diff --git a/src/osw/tools/user_sync/sources.py b/src/osw/tools/user_sync/sources.py new file mode 100644 index 00000000..24f0072c --- /dev/null +++ b/src/osw/tools/user_sync/sources.py @@ -0,0 +1,294 @@ +"""External data sources for the user-item sync tool. + +This module wraps the MediaWiki ``allusers`` API to list accounts and contains a +client for the public ORCID API that fetches and parses ORCID records. +""" + +from __future__ import annotations + +import re +import time +from dataclasses import dataclass, field +from typing import Any, Dict, Iterable, Iterator, List, Optional, Sequence + +import requests + +from .config import ORCID_API_BASE_DEFAULT + +# An ORCID iD is 16 digits in groups of four; the last character may be an X. +ORCID_RE = re.compile(r"^\d{4}-\d{4}-\d{4}-\d{3}[\dX]$") + +_ALLUSERS_PROPS = "registration|editcount|groups|centralids|blockinfo" + +# MediaWiki default reserved system usernames. These ship with MediaWiki and are +# the same across instances, so skipping them is not a per-instance skip-list. +RESERVED_USERNAMES = frozenset({ + "MediaWiki default", + "Maintenance script", + "Conversion script", + "Template namespace initialisation script", + "ScriptImporter", + "Delete page script", + "Move page script", + "Command line script", + "Unknown user", + "MediaWiki message delivery", + "Flow talk page manager", + "Abuse filter", + "New user message", +}) + + +def is_orcid_username(name: str) -> bool: + """True if a MediaWiki username is an ORCID iD (whitelisted ORCID login).""" + return bool(ORCID_RE.match(name or "")) + + +@dataclass +class MwUser: + """A MediaWiki account and the metadata exposed by the ``allusers`` API.""" + + name: str + userid: Optional[int] = None + registration: Optional[str] = None + editcount: int = 0 + groups: List[str] = field(default_factory=list) + blocked: bool = False + + @property + def is_orcid(self) -> bool: + return is_orcid_username(self.name) + + @property + def orcid_id(self) -> Optional[str]: + """The bare ORCID iD when the username is one, else None.""" + return self.name if self.is_orcid else None + + @classmethod + def from_api(cls, raw: Dict[str, Any]) -> MwUser: + return cls( + name=raw["name"], + userid=raw.get("userid"), + registration=raw.get("registration") or None, + editcount=int(raw.get("editcount") or 0), + groups=list(raw.get("groups") or []), + blocked="blockid" in raw, + ) + + +def _iter_allusers(site: Any, batch: Any = "max") -> Iterator[Dict[str, Any]]: + """Yield raw ``allusers`` records, following API continuation.""" + params: Dict[str, Any] = { + "list": "allusers", + "auprop": _ALLUSERS_PROPS, + "aulimit": batch, + } + while True: + resp = site.api("query", **params) + yield from resp.get("query", {}).get("allusers", []) + cont = resp.get("continue") + if not cont: + break + params.update(cont) + + +def enumerate_mw_users( + site: Any, + excluded_groups: Sequence[str] = ("bot",), + excluded_usernames: Sequence[str] = (), + include_orcid: bool = True, + include_non_orcid: bool = True, + limit: Optional[int] = None, + batch: Any = "max", + exclude_blocked: bool = True, + skipped_blocked: Optional[List[str]] = None, +) -> List[MwUser]: + """List MediaWiki accounts to sync. + + Args: + site: An object exposing ``api("query", ...)`` (an mwclient Site). + excluded_groups: Accounts in any of these groups are skipped (bots, admins), + unless the username is an exact ORCID iD (a person's own account). + Bot accounts derived from an ORCID (e.g. ``<orcid>@bot``) are not exact + ORCID iDs and stay excluded. + excluded_usernames: Exact usernames to skip (reserved system accounts). + include_orcid: If False, skip ORCID-username accounts. + include_non_orcid: If False, keep only ORCID-username accounts. + limit: Keep at most this many accounts after filtering. + batch: ``aulimit`` value passed to the API. + exclude_blocked: If True, skip blocked accounts, ORCID or not. + skipped_blocked: If given, receives the names of the skipped blocked + accounts. + """ + excluded = set(excluded_groups) + excluded_names = set(excluded_usernames) + users: List[MwUser] = [] + for raw in _iter_allusers(site, batch=batch): + user = MwUser.from_api(raw) + if user.name in excluded_names: + continue + if exclude_blocked and user.blocked: + if skipped_blocked is not None: + skipped_blocked.append(user.name) + continue + if not user.is_orcid and excluded.intersection(user.groups): + continue + if not include_non_orcid and not user.is_orcid: + continue + if not include_orcid and user.is_orcid: + continue + users.append(user) + if limit is not None and len(users) >= limit: + break + return users + + +def partition_by_orcid(users: Iterable[MwUser]) -> Dict[str, List[MwUser]]: + """Split users into ``{"orcid": [...], "other": [...]}``.""" + orcid: List[MwUser] = [] + other: List[MwUser] = [] + for user in users: + (orcid if user.is_orcid else other).append(user) + return {"orcid": orcid, "other": other} + + +class OrcidRateLimitError(RuntimeError): + """Raised when the ORCID public API keeps returning HTTP 429.""" + + +@dataclass +class OrcidAffiliation: + """An employment entry from an ORCID record.""" + + organization: str + ror_id: Optional[str] = None + department: Optional[str] = None + role: Optional[str] = None + + +@dataclass +class OrcidProfile: + """The subset of a public ORCID record that maps to an OSW User item.""" + + orcid_id: str + given_names: Optional[str] = None + family_name: Optional[str] = None + credit_name: Optional[str] = None + other_names: List[str] = field(default_factory=list) + emails: List[str] = field(default_factory=list) + urls: List[str] = field(default_factory=list) + affiliations: List[OrcidAffiliation] = field(default_factory=list) + + @property + def orcid_uri(self) -> str: + return f"https://orcid.org/{self.orcid_id}" + + @property + def display_name(self) -> Optional[str]: + """Best full name: credit name, else given + family, else None.""" + if self.credit_name: + return self.credit_name + parts = [p for p in (self.given_names, self.family_name) if p] + return " ".join(parts) or None + + +def _value(node: Any, *keys: str) -> Optional[Any]: + """Safely walk nested dicts, returning None on any missing key.""" + cur = node + for key in keys: + if not isinstance(cur, dict): + return None + cur = cur.get(key) + return cur + + +def _parse_affiliations(activities: Any) -> List[OrcidAffiliation]: + affiliations: List[OrcidAffiliation] = [] + groups = _value(activities, "employments", "affiliation-group") or [] + for group in groups: + for summary in group.get("summaries") or []: + emp = summary.get("employment-summary") or {} + org = emp.get("organization") or {} + name = org.get("name") + if not name: + continue + ror = None + disamb = org.get("disambiguated-organization") or {} + if (disamb.get("disambiguation-source") or "").upper() == "ROR": + ror = disamb.get("disambiguated-organization-identifier") + affiliations.append( + OrcidAffiliation( + organization=name, + ror_id=ror, + department=emp.get("department-name"), + role=emp.get("role-title"), + ) + ) + return affiliations + + +def parse_orcid_record(data: Dict[str, Any], orcid_id: str) -> OrcidProfile: + """Parse an ORCID ``/record`` JSON payload into an OrcidProfile (pure).""" + person = data.get("person") or {} + other_names = [ + n.get("content") + for n in _value(person, "other-names", "other-name") or [] + if n.get("content") + ] + emails = [ + e.get("email") + for e in _value(person, "emails", "email") or [] + if e.get("email") + ] + urls = [ + _value(u, "url", "value") + for u in _value(person, "researcher-urls", "researcher-url") or [] + if _value(u, "url", "value") + ] + return OrcidProfile( + orcid_id=orcid_id, + given_names=_value(person, "name", "given-names", "value"), + family_name=_value(person, "name", "family-name", "value"), + credit_name=_value(person, "name", "credit-name", "value"), + other_names=other_names, + emails=emails, + urls=urls, + affiliations=_parse_affiliations(data.get("activities-summary") or {}), + ) + + +def fetch_orcid_record( + orcid_id: str, + session: Optional[requests.Session] = None, + base: str = ORCID_API_BASE_DEFAULT, + timeout: float = 30.0, + max_retries: int = 2, + cache: Optional[Dict[str, Optional[OrcidProfile]]] = None, +) -> Optional[OrcidProfile]: + """Fetch and parse a public ORCID record. + + Returns None when the record does not exist (HTTP 404). Retries on HTTP 429 + honoring ``Retry-After`` and raises OrcidRateLimitError if still limited. + """ + if cache is not None and orcid_id in cache: + return cache[orcid_id] + sess = session or requests.Session() + url = f"{base}/{orcid_id}/record" + headers = {"Accept": "application/json"} + result: Optional[OrcidProfile] = None + for attempt in range(max_retries + 1): + resp = sess.get(url, headers=headers, timeout=timeout) + if resp.status_code == 404: + result = None + break + if resp.status_code == 429: + if attempt >= max_retries: + raise OrcidRateLimitError(f"ORCID rate limit for {orcid_id}") + time.sleep(float(resp.headers.get("Retry-After", 1))) + continue + resp.raise_for_status() + result = parse_orcid_record(resp.json(), orcid_id) + break + if cache is not None: + cache[orcid_id] = result + return result diff --git a/src/osw/tools/user_sync/sync.py b/src/osw/tools/user_sync/sync.py new file mode 100644 index 00000000..2cfeb0a2 --- /dev/null +++ b/src/osw/tools/user_sync/sync.py @@ -0,0 +1,324 @@ +"""Orchestrator for the user-item sync tool. + +Ties the sources, mapping, reconciliation, interactive and store steps into one +run: enumerate MediaWiki users, enrich ORCID users, reconcile against existing +items, preview and confirm, then store items, redirects and organizations and +verify. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field, replace +from typing import Any, Dict, FrozenSet, List, Optional, Tuple + +from osw.core import OSW + +from .build import apply_update, build_organization, build_user +from .config import SyncConfig +from .existing import adopt_existing_pages, load_existing_users +from .interactive import Prompter, Resolution, resolve_plan +from .mapping import ProposedOrganization, map_user +from .prune import clear_jsondata_keys +from .reconcile import REMOVE, existing_fields, reconcile +from .redirects import ensure_redirect +from .sources import ( + RESERVED_USERNAMES, + OrcidRateLimitError, + enumerate_mw_users, + fetch_orcid_record, +) + + +@dataclass +class SyncReport: + """Outcome of a sync run.""" + + created: List[str] = field(default_factory=list) + updated: List[str] = field(default_factory=list) + skipped: List[str] = field(default_factory=list) + failed: Dict[str, str] = field(default_factory=dict) + pruned: List[str] = field(default_factory=list) + redirects_created: List[str] = field(default_factory=list) + organizations: List[str] = field(default_factory=list) + missing_email: List[str] = field(default_factory=list) + accepted: bool = False + + def summary(self) -> str: + state = "applied" if self.accepted else "no changes written" + return ( + f"{state}: created={len(self.created)} updated={len(self.updated)} " + f"skipped={len(self.skipped)} pruned={len(self.pruned)} " + f"failed={len(self.failed)} " + f"redirects={len(self.redirects_created)} orgs={len(self.organizations)} " + f"missing_email={len(self.missing_email)}" + ) + + +def _build_proposals(config: SyncConfig, osw: Any, session: Any): + """Enumerate MediaWiki users, enrich ORCID users and map to proposals. + + Returns the proposed users, the organizations and the names of the blocked + accounts that were skipped. + """ + excluded = tuple(config.excluded_groups) if config.exclude_bot_group else () + reserved = RESERVED_USERNAMES if config.exclude_system_usernames else () + skipped_blocked: List[str] = [] + mw_users = enumerate_mw_users( + osw.mw_site, + excluded_groups=excluded, + excluded_usernames=reserved, + include_orcid=config.include_orcid, + include_non_orcid=config.include_non_orcid, + limit=config.limit, + exclude_blocked=not config.include_blocked, + skipped_blocked=skipped_blocked, + ) + cache: Dict[str, Any] = {} + proposed_users = [] + org_map: Dict[Any, ProposedOrganization] = {} + for mw_user in mw_users: + profile = None + if mw_user.is_orcid: + try: + profile = fetch_orcid_record( + mw_user.orcid_id, + session=session, + base=config.orcid_api_base, + cache=cache, + ) + except OrcidRateLimitError: + profile = None + proposed, orgs = map_user( + mw_user, + profile, + include_websites=config.include_websites, + link_organizations=config.link_organizations, + ) + proposed_users.append(proposed) + for org in orgs: + org_map.setdefault(org.uuid, org) + return proposed_users, org_map, skipped_blocked + + +def _record_partial(report: SyncReport, exc: Exception) -> List[str]: + """Record each failed title of a partial store; return the stored titles.""" + for title, error in exc.failed.items(): + report.failed[f"store:{title}"] = str(error) + return list(exc.stored) + + +def _store_organizations(osw: Any, org_map, report: SyncReport) -> None: + if not org_map: + return + entities = [build_organization(org) for org in org_map.values()] + try: + osw.store_entity( + OSW.StoreEntityParam(entities=entities, edit_comment="user-sync: orgs") + ) + report.organizations = [org.full_page_title for org in org_map.values()] + except OSW.StoreEntityPartialError as exc: + report.organizations = _record_partial(report, exc) + except Exception as exc: # pragma: no cover - network failure path + report.failed["organizations"] = str(exc) + + +def _store_users( + osw: Any, + resolution: Resolution, + report: SyncReport, + failed_orgs: FrozenSet[str] = frozenset(), +) -> None: + """Store created and updated items; count a title only once it is stored. + + New links to an organization in ``failed_orgs`` are dropped, so no item + points to an organization that was not stored. A link the existing item + already has is kept. A re-run proposes the dropped links again. + """ + entities = [] + pending: List[Tuple[str, str]] = [] # (action, title) per entity + for resolved in resolution.resolved: + proposed = resolved.change.proposed + drop = set(proposed.organizations) & failed_orgs + if drop and resolved.change.existing is not None: + drop -= existing_fields(resolved.change.existing)["organizations"] + if drop: + proposed = replace( + proposed, + organizations=[o for o in proposed.organizations if o not in drop], + ) + try: + if resolved.action == "create": + entities.append(build_user(proposed)) + elif resolved.action == "update": + entities.append( + apply_update( + resolved.change.existing, proposed, resolved.apply_fields + ) + ) + else: + report.skipped.append(proposed.full_page_title) + continue + pending.append((resolved.action, proposed.full_page_title)) + except Exception as exc: # pragma: no cover - build failure path + report.failed[f"build:{proposed.username}"] = str(exc) + if not entities: + return + try: + osw.store_entity( + OSW.StoreEntityParam( + entities=entities, + overwrite=True, + edit_comment="user-sync", + ) + ) + stored = {title for _, title in pending} + except OSW.StoreEntityPartialError as exc: + stored = set(_record_partial(report, exc)) + except Exception as exc: # pragma: no cover - network failure path + report.failed["store"] = str(exc) + stored = set() + for action, title in pending: + if title in stored: + (report.created if action == "create" else report.updated).append(title) + + +# Names of reconciled fields mapped to their keys in the item's jsondata. +_JSONDATA_KEYS = { + "websites": "website", + "organizations": "organization", + "employment_contract_status": "employment_contract_status", +} + + +def _prune_users(osw: Any, resolution: Resolution, report: SyncReport) -> None: + """Clear removed fields of stored existing items (see clear_jsondata_keys).""" + updated = set(report.updated) + for resolved in resolution.resolved: + if resolved.action != "update": + continue + if resolved.change.proposed.full_page_title not in updated: + continue # the update was not stored + keys = [ + _JSONDATA_KEYS[d.name] + for d in resolved.change.diffs + if d.status == REMOVE + and d.name in resolved.apply_fields + and d.name in _JSONDATA_KEYS + ] + if not keys: + continue + title = resolved.change.proposed.full_page_title + try: + if clear_jsondata_keys(osw, title, keys): + report.pruned.append(title) + except Exception as exc: # pragma: no cover - network failure path + report.failed[f"prune:{title}"] = str(exc) + + +def _create_redirects(osw: Any, resolution: Resolution, report: SyncReport) -> None: + # A redirect needs an existing target: an item stored in this run, or one + # that existed before it (update or skip). A failed create gets none; a + # re-run writes it once the item is stored. + created = set(report.created) + for resolved in resolution.resolved: + proposed = resolved.change.proposed + if resolved.action == "create" and proposed.full_page_title not in created: + continue + try: + written = ensure_redirect(osw, proposed.username, proposed.full_page_title) + if written: + report.redirects_created.append(written) + except Exception as exc: # pragma: no cover - network failure path + report.failed[f"redirect:{proposed.username}"] = str(exc) + + +def _warn_missing_email(prompter: Prompter, missing: List[str]) -> None: + if missing: + prompter.write( + f"WARNING: {len(missing)} user(s) have no ORCID email: " + + ", ".join(missing) + ) + + +def _verify(osw: Any, report: SyncReport) -> None: + titles = report.created + report.updated + if not titles: + return + try: + loaded = osw.load_entity(titles) + loaded_list = loaded if isinstance(loaded, list) else [loaded] + found = {getattr(e, "username", None) for e in loaded_list} + missing = [ + t + for t, e in zip(titles, loaded_list) + if getattr(e, "username", None) is None + ] + if missing or None in found: + report.failed["verify"] = f"missing username on {missing}" + except Exception as exc: # pragma: no cover - network failure path + report.failed["verify"] = str(exc) + + +def run_user_sync( + config: SyncConfig, + osw: Optional[OSW] = None, + prompter: Optional[Prompter] = None, + session: Any = None, +) -> SyncReport: + """Create or update OSW User items from MediaWiki accounts and ORCID data.""" + if osw is None: + raise ValueError("run_user_sync requires an authenticated OSW connection") + prompter = prompter or Prompter() + + proposed_users, org_map, skipped_blocked = _build_proposals(config, osw, session) + missing_email = [p.username for p in proposed_users if not p.emails] + existing = load_existing_users(osw) + adopt_existing_pages(osw, existing, proposed_users) + plan = reconcile( + proposed_users, + existing, + enabled_fields=frozenset(config.enabled_optional_fields()), + prune=config.prune, + ) + + summary_lines = [] + if skipped_blocked: + summary_lines.append( + f"Skipped blocked accounts ({len(skipped_blocked)}): " + + ", ".join(skipped_blocked) + ) + if config.link_organizations and org_map: + summary_lines.append(f"Organizations to ensure: {len(org_map)}") + if config.create_redirects: + summary_lines.append("Redirects: User:<username> -> item for created/updated") + + resolution = resolve_plan( + plan, + prompter, + dry_run=config.dry_run, + auto_apply=config.auto_apply, + summary_lines=summary_lines, + ) + + report = SyncReport(accepted=resolution.accepted, missing_email=missing_email) + if not resolution.accepted: + _warn_missing_email(prompter, missing_email) + return report + + failed_orgs: FrozenSet[str] = frozenset() + if config.link_organizations: + _store_organizations(osw, org_map, report) + stored_orgs = set(report.organizations) + failed_orgs = frozenset( + o.full_page_title + for o in org_map.values() + if o.full_page_title not in stored_orgs + ) + _store_users(osw, resolution, report, failed_orgs=failed_orgs) + if config.prune and "store" not in report.failed: + _prune_users(osw, resolution, report) + if config.create_redirects: + _create_redirects(osw, resolution, report) + _verify(osw, report) + _warn_missing_email(prompter, missing_email) + return report diff --git a/src/osw/utils/workflow.py b/src/osw/utils/workflow.py index 7618889d..e8ae51fa 100644 --- a/src/osw/utils/workflow.py +++ b/src/osw/utils/workflow.py @@ -560,7 +560,10 @@ async def register_flow( description=[model.Description(text=flow.description or "")], flow_id=str(flow_uuid), hosted_software=[get_full_title(this_tool)], - domain=prefect_domain, + # None rather than "" when the hostname is unknown, like url_path above: + # an explicitly assigned empty value is written and would clear a domain + # already stored on the wiki + domain=prefect_domain or None, schema_=prefect_schema, network_port=prefect_port, url_path=prefect_path, diff --git a/src/osw/wtsite.py b/src/osw/wtsite.py index 67ce2e04..a2f9d884 100644 --- a/src/osw/wtsite.py +++ b/src/osw/wtsite.py @@ -1060,6 +1060,13 @@ def create_page_package(self, param: CreatePagePackageParam): WtSite.GetPageParam(titles=titles_to_fetch, parallel=param.parallel) ).pages ) + + # get_page appends in completion order when fetching in parallel, so the + # page that claims a file referenced by several pages would vary between + # runs. Restore the configured order. + title_rank = {title: rank for rank, title in enumerate(added_titles)} + pages.sort(key=lambda p: (title_rank.get(p.title, len(title_rank)), p.title)) + added_file_titles = [] page_dumps = {} page_files = {} @@ -1084,8 +1091,10 @@ def create_page_package(self, param: CreatePagePackageParam): if debug and len(ignored_files_pages) > 0: _logger.debug(f"Ignored: {ignored_files_pages}") referenced_file_pages = included_file_pages - # find those files that are not already in the package - page_files[page.title] = list( + # find those files that are not already in the package. sorted() + # because the set difference and find_file_page_refs_in_slots + # both return an order that varies between runs + page_files[page.title] = sorted( set(referenced_file_pages) - set(added_file_titles) ) added_file_titles = list(set(added_file_titles + referenced_file_pages)) diff --git a/tests/fixtures/category_template_selection/after-form-editor-save.json b/tests/fixtures/category_template_selection/after-form-editor-save.json new file mode 100644 index 00000000..58840d7d --- /dev/null +++ b/tests/fixtures/category_template_selection/after-form-editor-save.json @@ -0,0 +1,568 @@ +{ + "Category:OSW37a57741ae2e4dd4b29b1172b6848be8": { + "jsondata": { + "subclass_of": [ + "Category:Item" + ], + "type": [ + "Category:Category" + ], + "version": "0.1.0", + "label": [ + { + "text": "Physical Item", + "lang": "en" + }, + { + "text": "Physischer Gegenstand", + "lang": "de" + } + ], + "characteristics": [ + "Category:OSW42d82cfc0c304ca48b21e72557a5472f" + ], + "uuid": "37a57741-ae2e-4dd4-b29b-1172b6848be8", + "description": [ + { + "text": "Physical item as opposed to intangible item. It can posses properties such as position, geometry, weight, volume , etc. ", + "lang": "en" + }, + { + "text": "Physischer Gegenstand im Gegensatz zu immateriellen Gegenständen. Er kann Eigenschaften wie Position, Geometrie, Gewicht, Volumen usw. besitzen. ", + "lang": "de" + } + ], + "name": "PhysicalItem", + "metaclass": [ + "Category:OSWca99a9bb9ad24386b222d18a73c2af5e" + ], + "utf8_icon": [ + "🪨" + ] + }, + "jsonschema": { + "@context": [ + "/wiki/Category:Item?action=raw&slot=jsonschema", + {}, + "/wiki/Category:OSW42d82cfc0c304ca48b21e72557a5472f?action=raw&slot=jsonschema", + {} + ], + "allOf": [ + { + "$ref": "#/$defs/generated" + } + ], + "title": "PhysicalItem", + "title*": { + "en": "Physical Item", + "de": "Physischer Gegenstand" + }, + "description": "Physical item as opposed to intangible item. It can posses properties such as position, geometry, weight, volume , etc. ", + "description*": { + "en": "Physical item as opposed to intangible item. It can posses properties such as position, geometry, weight, volume , etc. ", + "de": "Physischer Gegenstand im Gegensatz zu immateriellen Gegenständen. Er kann Eigenschaften wie Position, Geometrie, Gewicht, Volumen usw. besitzen. " + }, + "properties": { + "quantity_statements": { + "type": "array", + "format": "tabs", + "title": "Quantitative statement(s)", + "title*": { + "de": "Quantitative Aussage(n)" + }, + "eval_template": { + "type": "mustache-wikitext", + "mode": "render", + "value": "{{#quantity_statements}}.. {{#quantity}}[[{{quantity}}]]{{/quantity}} {{#value}}{{value}}{{/value}}<br>{{/quantity_statements}}" + }, + "items": { + "$ref": "/wiki/JsonSchema:QuantityProperty?action=raw" + } + }, + "origin": { + "type": "array", + "format": "table", + "title": "Origin", + "title*": { + "de": "Ursprung" + }, + "description": "Other physical item(s) that this physical item was derived from.", + "description*": { + "de": "Andere(r) physische(r) Gegenstand/Gegenstände aus der dieser physische Gegenstand hergestellt wurde." + }, + "options": { + "enable_array_copy": true, + "array_copy_ignore": [ + "uuid", + "attachments" + ] + }, + "eval_template": [ + { + "type": "mustache-wikitext", + "mode": "render", + "value": "{{=<% %>=}} <ul><%#origin%><li>[[<%origin_item_instance%>]] ({{#switch:{{USERLANGUAGECODE}}|en=Type|de=Typ|#default=Type}}: {{#ask:[[<%origin_item_instance%>]]|?HasType=|format=plain|mainlabel=-}})</li><%/origin%></ul>" + } + ], + "items": { + "type": "object", + "format": "grid-strict", + "title": "Other physical Item", + "title*": { + "de": "Anderer physischer Gegenstand" + }, + "id": "origin_item", + "headerTemplate": "{{ i1 }}", + "defaultProperties": [], + "required": [ + "uuid", + "origin_item_type", + "origin_item_instance_property", + "origin_item_instance" + ], + "properties": { + "uuid": { + "type": "string", + "dynamic_template": "{{_uuid_}}", + "options": { + "hidden": true, + "dynamic_template": { + "override": "empty" + } + } + }, + "origin_item_type": { + "title": "Type", + "title*": { + "de": "Typ" + }, + "type": "string", + "format": "autocomplete", + "range": "Category:Category", + "options": { + "autocomplete": { + "$comment": "Tool", + "query": "[[:Category:OSW37a57741ae2e4dd4b29b1172b6848be8]]OR[[SubClassOf::Category:OSW37a57741ae2e4dd4b29b1172b6848be8]]OR[[SubClassOf.SubClassOf::Category:OSW37a57741ae2e4dd4b29b1172b6848be8]]OR[[SubClassOf.SubClassOf.SubClassOf::Category:OSW37a57741ae2e4dd4b29b1172b6848be8]]OR[[SubClassOf.SubClassOf.SubClassOf.SubClassOf::Category:OSW37a57741ae2e4dd4b29b1172b6848be8]]OR[[SubClassOf.SubClassOf.SubClassOf.SubClassOf.SubClassOf::Category:OSW37a57741ae2e4dd4b29b1172b6848be8]]" + }, + "grid_columns": 12 + }, + "default": "Category:OSW37a57741ae2e4dd4b29b1172b6848be8", + "propertyOrder": 40 + }, + "origin_item_instance_property": { + "type": "string", + "default": "Property:HasOrigin", + "options": { + "hidden": true + } + }, + "origin_item_instance": { + "title": "Instance", + "title*": { + "de": "Instanz" + }, + "type": "string", + "format": "autocomplete", + "range": "Category:OSW37a57741ae2e4dd4b29b1172b6848be8", + "watch": { + "origin_item_type_": "origin_item.origin_item_type" + }, + "options": { + "autocomplete": { + "query": "[[HasType::$(origin_item_type_)]]|?Display_title_of=label|?HasImage=image|?HasDescription=description" + }, + "grid_columns": 12 + }, + "propertyOrder": 50 + } + } + } + }, + "type_no": { + "allOf": [ + { + "$ref": "/wiki/JsonSchema:TypeNumber?action=raw&slot=main" + } + ] + }, + "manufacturer": { + "allOf": [ + { + "$ref": "/wiki/JsonSchema:Manufacturer?action=raw&slot=main" + } + ] + }, + "data_sheet": { + "$ref": "/wiki/JsonSchema:DataSheet?action=raw&slot=main" + }, + "safety_data_sheet": { + "$ref": "/wiki/JsonSchema:SafetyDataSheet?action=raw&slot=main" + }, + "manual": { + "$ref": "/wiki/JsonSchema:Manual?action=raw&slot=main" + }, + "instructions": { + "$ref": "/wiki/JsonSchema:Instructions?action=raw&slot=main" + } + }, + "$defs": { + "generated": { + "$comment": "Autogenerated section - do not edit. Generated from Category:Category Category:OSWffe74f291d354037b318c422591c5023", + "allOf": [ + { + "$ref": "/wiki/Category:Item?action=raw&slot=jsonschema" + }, + { + "$ref": "/wiki/Category:OSW42d82cfc0c304ca48b21e72557a5472f?action=raw&slot=jsonschema" + } + ], + "type": "object", + "uuid": "37a57741-ae2e-4dd4-b29b-1172b6848be8", + "x-oold-version": "0.1.0", + "title": "GeneratedPhysicalItem", + "title*": { + "en": "Physical Item", + "de": "Physischer Gegenstand" + }, + "description": "This is an autogenerated partial class definition of 'PhysicalItem'", + "description*": { + "en": "Physical item as opposed to intangible item. It can posses properties such as position, geometry, weight, volume , etc. ", + "de": "Physischer Gegenstand im Gegensatz zu immateriellen Gegenständen. Er kann Eigenschaften wie Position, Geometrie, Gewicht, Volumen usw. besitzen. " + }, + "required": [ + "type" + ], + "properties": { + "type": { + "default": [ + "Category:OSW37a57741ae2e4dd4b29b1172b6848be8" + ] + } + }, + "defaultProperties": [ + "type" + ] + } + } + }, + "revision": { + "revid": 134604, + "parentid": 129378, + "user": "Lukas Gold", + "timestamp": "2026-11-05T12:52:22Z", + "comment": "" + } + }, + "Category:OSWca99a9bb9ad24386b222d18a73c2af5e": { + "jsondata": { + "subclass_of": [ + "Category:Category" + ], + "type": [ + "Category:Category" + ], + "version": "0.1.0", + "label": [ + { + "text": "Physical Item Type", + "lang": "en" + } + ], + "uuid": "ca99a9bb-9ad2-4386-b222-d18a73c2af5e", + "description": [ + { + "text": "Helps to create subcategories of PhysicalItem", + "lang": "en" + } + ], + "name": "PhysicalItemType" + }, + "jsonschema": { + "@context": [ + "/wiki/Category:Category?action=raw&slot=jsonschema", + {}, + { + "constituents": { + "@type": "@id", + "@id": "Property:HasStatement", + "@context": { + "property": "Property:HasProperty", + "type": "Property:HasValue" + } + }, + "data_sheet": { + "@id": "Property:HasDataSheet", + "@type": "@id" + }, + "instructions": { + "@id": "Property:HasInstructions", + "@type": "@id" + }, + "manual": { + "@id": "Property:HasManual", + "@type": "@id" + }, + "manufacturer": { + "@id": "Property:HasManufacturer", + "@type": "@id" + }, + "safety_data_sheet": { + "@id": "Property:HasSafetyDataSheet", + "@type": "@id" + }, + "type_no": "Property:HasTypeNumber" + } + ], + "allOf": [ + { + "$ref": "#/$defs/generated" + } + ], + "title": "PhysicalItemType", + "title*": { + "en": "Physical Item Type" + }, + "description": "Helps to create subcategories of PhysicalItem", + "description*": { + "en": "Helps to create subcategories of PhysicalItem" + }, + "defaultPropertis": [ + "type_no", + "manufacturer", + "data_sheet", + "safety_data_sheet", + "supply", + "specifications" + ], + "properties": { + "type_no": { + "$ref": "/wiki/JsonSchema:TypeNumber?action=raw&slot=main" + }, + "manufacturer": { + "$ref": "/wiki/JsonSchema:Manufacturer?action=raw&slot=main" + }, + "data_sheet": { + "$ref": "/wiki/JsonSchema:DataSheet?action=raw&slot=main" + }, + "safety_data_sheet": { + "$ref": "/wiki/JsonSchema:SafetyDataSheet?action=raw&slot=main" + }, + "instructions": { + "$ref": "/wiki/JsonSchema:Instructions?action=raw&slot=main" + }, + "manual": { + "$ref": "/wiki/JsonSchema:Manual?action=raw&slot=main" + }, + "supply": { + "title": "Supply", + "title*": { + "de": "Bezugsmöglichkeit" + }, + "description": "Supply of the material with supplier, article no. and link", + "description*": { + "de": "Bezugsmöglichkeit des Materials mit Lieferant, Artikelnummer und Link" + }, + "type": "array", + "format": "table", + "items": { + "title": "Supply", + "title*": { + "de": "Bezugsmöglichkeit" + }, + "type": "object", + "properties": { + "supplier": { + "title": "Supplier", + "title*": { + "de": "Lieferant" + }, + "type": "string", + "format": "autocomplete", + "options": { + "autocomplete": { + "category": "Category:OSW3cb8cef2225e403092f098f99bc4c472" + } + } + }, + "article_no": { + "title": "Article no.", + "title*": { + "de": "Artikelnummer" + }, + "description": "Article number in the catalogue of the supplier", + "description*": { + "de": "Artikelnummer im Katalog des Lieferanten" + }, + "type": "string" + }, + "comment": { + "title": "Comment", + "title*": { + "de": "Kommentar" + }, + "description": "Comment on the supply of the material", + "description*": { + "de": "Kommentar bezüglich der Bezugsmöglichkeit des Materials" + }, + "type": "string", + "format": "" + }, + "link": { + "title": "Link", + "title*": { + "de": "Link" + }, + "description": "Link to the article in the online catalogue of the supplier", + "description*": { + "de": "Link zum Artikel im Online-Katalog des Lieferanten" + }, + "type": "string", + "format": "uri" + } + } + } + }, + "specifications": { + "title": "Specification of technical properties", + "title*": { + "de": "Spezifikation technischer Eigenschaften" + }, + "type": "array", + "format": "table", + "items": { + "title": "Attribute", + "title*": { + "de": "Attribut" + }, + "type": "object", + "headerTemplate": "{{i1}} - {{self.name}}", + "required": [ + "attribute_name", + "quantity_prop" + ], + "properties": { + "attribute_name": { + "title": "Name", + "type": "array", + "format": "table", + "items": { + "$ref": "/wiki/JsonSchema:Label?action=raw" + }, + "minItems": 1 + }, + "quantity_prop": { + "$ref": "/wiki/JsonSchema:QuantityProperty?action=raw" + } + } + } + }, + "constituents": { + "type": "array", + "format": "tabs", + "title": "Constituents", + "title*": { + "de": "Bestandteile" + }, + "description": "Components or parts", + "items": { + "type": "object", + "title": "Constituent", + "title*": { + "de": "Bestandteil" + }, + "headerTemplate": "{{i1}} - {{self.name}}", + "required": [ + "name", + "type", + "uuid", + "property" + ], + "properties": { + "uuid": { + "title": "UUID", + "type": "string", + "format": "uuid", + "options": { + "hidden": true + } + }, + "property": { + "type": "string", + "default": "Property:HasPart", + "options": { + "hidden": true + } + }, + "is_array": { + "title": "Is array", + "type": "boolean" + }, + "name": { + "title": "Name", + "type": "string" + }, + "type": { + "title": "Type", + "type": "string", + "format": "autocomplete", + "$comment": "Material", + "range": "Category:OSW31ca9a739cb24079b36824045c0832aa", + "options": { + "autocomplete": { + "$comment": "todo: we need an any-subcategory-of shortcut", + "query": "[[SubClassOf::Category:OSW31ca9a739cb24079b36824045c0832aa]]OR[[SubClassOf.SubClassOf::Category:OSW31ca9a739cb24079b36824045c0832aa]]OR[[SubClassOf.SubClassOf.SubClassOf::Category:OSW31ca9a739cb24079b36824045c0832aa]]OR[[SubClassOf.SubClassOf.SubClassOf.SubClassOf::Category:OSW31ca9a739cb24079b36824045c0832aa]]OR[[SubClassOf.SubClassOf.SubClassOf.SubClassOf.SubClassOf::Category:OSW31ca9a739cb24079b36824045c0832aa]]" + } + } + }, + "default": { + "title": "Default value", + "type": "string" + } + } + } + } + }, + "$defs": { + "generated": { + "$comment": "Autogenerated section - do not edit. Generated from Category:Category Category:OSWffe74f291d354037b318c422591c5023", + "allOf": [ + { + "$ref": "/wiki/Category:Category?action=raw&slot=jsonschema" + } + ], + "type": "object", + "uuid": "ca99a9bb-9ad2-4386-b222-d18a73c2af5e", + "x-oold-version": "0.1.0", + "title": "GeneratedPhysicalItemType", + "title*": { + "en": "Physical Item Type" + }, + "description": "This is an autogenerated partial class definition of 'PhysicalItemType'", + "description*": { + "en": "Helps to create subcategories of PhysicalItem" + }, + "required": [ + "type" + ], + "properties": { + "type": { + "default": [ + "Category:OSWca99a9bb9ad24386b222d18a73c2af5e" + ] + } + }, + "defaultProperties": [ + "type" + ] + } + } + }, + "revision": { + "revid": 134605, + "parentid": 122162, + "user": "Lukas Gold", + "timestamp": "2026-11-05T12:52:46Z", + "comment": "" + } + } +} diff --git a/tests/fixtures/category_template_selection/before-form-editor-save.json b/tests/fixtures/category_template_selection/before-form-editor-save.json new file mode 100644 index 00000000..1b816fc2 --- /dev/null +++ b/tests/fixtures/category_template_selection/before-form-editor-save.json @@ -0,0 +1,586 @@ +{ + "Category:OSW37a57741ae2e4dd4b29b1172b6848be8": { + "jsondata": { + "subclass_of": [ + "Category:Item" + ], + "type": [ + "Category:Category" + ], + "label": [ + { + "text": "Physical Item", + "lang": "en" + }, + { + "text": "Physischer Gegenstand", + "lang": "de" + } + ], + "uuid": "37a57741-ae2e-4dd4-b29b-1172b6848be8", + "description": [ + { + "text": "Physical item as opposed to intangible item. It can posses properties such as position, geometry, weight, volume , etc. ", + "lang": "en" + }, + { + "text": "Physischer Gegenstand im Gegensatz zu immateriellen Gegenständen. Er kann Eigenschaften wie Position, Geometrie, Gewicht, Volumen usw. besitzen. ", + "lang": "de" + } + ], + "name": "PhysicalItem", + "metaclass": [ + "Category:OSWca99a9bb9ad24386b222d18a73c2af5e" + ], + "utf8_icon": [ + "🪨" + ], + "characteristics": [ + "Category:OSW42d82cfc0c304ca48b21e72557a5472f" + ] + }, + "jsonschema": { + "@context": [ + "/wiki/Category:Item?action=raw&slot=jsonschema", + { + "data_sheet": { + "@id": "Property:HasDataSheet", + "@type": "@id" + }, + "instructions": { + "@id": "Property:HasInstructions", + "@type": "@id" + }, + "manual": { + "@id": "Property:HasManual", + "@type": "@id" + }, + "manufacturer": { + "@id": "Property:HasManufacturer", + "@type": "@id" + }, + "origin": { + "@id": "Property:HasStatement", + "@type": "@id", + "@context": { + "origin_item_instance_property": "Property:HasProperty", + "origin_item_instance": "Property:HasObject" + } + }, + "quantity_statements": { + "@id": "Property:HasStatement", + "@type": "@id" + }, + "safety_data_sheet": { + "@id": "Property:HasSafetyDataSheet", + "@type": "@id" + }, + "serial_number": "Property:HasSerialNumber", + "type_no": "Property:HasTypeNumber" + }, + "/wiki/Category:OSW42d82cfc0c304ca48b21e72557a5472f?action=raw&slot=jsonschema", + {} + ], + "allOf": [ + { + "$ref": "/wiki/Category:Item?action=raw&slot=jsonschema" + }, + { + "$ref": "#/$defs/generated" + } + ], + "type": "object", + "uuid": "37a57741-ae2e-4dd4-b29b-1172b6848be8", + "title": "PhysicalItem", + "title*": { + "en": "Physical Item", + "de": "Physischer Gegenstand" + }, + "description": "Physical item as opposed to intangible item. It can posses properties such as position, geometry, weight, volume , etc. ", + "description*": { + "en": "Physical item as opposed to intangible item. It can posses properties such as position, geometry, weight, volume , etc. ", + "de": "Physischer Gegenstand im Gegensatz zu immateriellen Gegenständen. Er kann Eigenschaften wie Position, Geometrie, Gewicht, Volumen usw. besitzen. " + }, + "required": [ + "type" + ], + "properties": { + "type": { + "default": [ + "Category:OSW37a57741ae2e4dd4b29b1172b6848be8" + ] + }, + "quantity_statements": { + "type": "array", + "format": "tabs", + "title": "Quantitative statement(s)", + "title*": { + "de": "Quantitative Aussage(n)" + }, + "eval_template": { + "type": "mustache-wikitext", + "mode": "render", + "value": "{{#quantity_statements}}.. {{#quantity}}[[{{quantity}}]]{{/quantity}} {{#value}}{{value}}{{/value}}<br>{{/quantity_statements}}" + }, + "items": { + "$ref": "/wiki/JsonSchema:QuantityProperty?action=raw" + } + }, + "origin": { + "type": "array", + "format": "table", + "title": "Origin", + "title*": { + "de": "Ursprung" + }, + "description": "Other physical item(s) that this physical item was derived from.", + "description*": { + "de": "Andere(r) physische(r) Gegenstand/Gegenstände aus der dieser physische Gegenstand hergestellt wurde." + }, + "options": { + "enable_array_copy": true, + "array_copy_ignore": [ + "uuid", + "attachments" + ] + }, + "eval_template": [ + { + "type": "mustache-wikitext", + "mode": "render", + "value": "{{=<% %>=}} <ul><%#origin%><li>[[<%origin_item_instance%>]] ({{#switch:{{USERLANGUAGECODE}}|en=Type|de=Typ|#default=Type}}: {{#ask:[[<%origin_item_instance%>]]|?HasType=|format=plain|mainlabel=-}})</li><%/origin%></ul>" + } + ], + "items": { + "type": "object", + "format": "grid-strict", + "title": "Other physical Item", + "title*": { + "de": "Anderer physischer Gegenstand" + }, + "id": "origin_item", + "headerTemplate": "{{ i1 }}", + "defaultProperties": [], + "required": [ + "uuid", + "origin_item_type", + "origin_item_instance_property", + "origin_item_instance" + ], + "properties": { + "uuid": { + "type": "string", + "dynamic_template": "{{_uuid_}}", + "options": { + "hidden": true, + "dynamic_template": { + "override": "empty" + } + } + }, + "origin_item_type": { + "title": "Type", + "title*": { + "de": "Typ" + }, + "type": "string", + "format": "autocomplete", + "range": "Category:Category", + "options": { + "autocomplete": { + "$comment": "Tool", + "query": "[[:Category:OSW37a57741ae2e4dd4b29b1172b6848be8]]OR[[SubClassOf::Category:OSW37a57741ae2e4dd4b29b1172b6848be8]]OR[[SubClassOf.SubClassOf::Category:OSW37a57741ae2e4dd4b29b1172b6848be8]]OR[[SubClassOf.SubClassOf.SubClassOf::Category:OSW37a57741ae2e4dd4b29b1172b6848be8]]OR[[SubClassOf.SubClassOf.SubClassOf.SubClassOf::Category:OSW37a57741ae2e4dd4b29b1172b6848be8]]OR[[SubClassOf.SubClassOf.SubClassOf.SubClassOf.SubClassOf::Category:OSW37a57741ae2e4dd4b29b1172b6848be8]]" + }, + "grid_columns": 12 + }, + "default": "Category:OSW37a57741ae2e4dd4b29b1172b6848be8", + "propertyOrder": 40 + }, + "origin_item_instance_property": { + "type": "string", + "default": "Property:HasOrigin", + "options": { + "hidden": true + } + }, + "origin_item_instance": { + "title": "Instance", + "title*": { + "de": "Instanz" + }, + "type": "string", + "format": "autocomplete", + "range": "Category:OSW37a57741ae2e4dd4b29b1172b6848be8", + "watch": { + "origin_item_type_": "origin_item.origin_item_type" + }, + "options": { + "autocomplete": { + "query": "[[HasType::$(origin_item_type_)]]|?Display_title_of=label|?HasImage=image|?HasDescription=description" + }, + "grid_columns": 12 + }, + "propertyOrder": 50 + } + } + } + }, + "type_no": { + "allOf": [ + { + "$ref": "/wiki/JsonSchema:TypeNumber?action=raw&slot=main" + } + ] + }, + "manufacturer": { + "allOf": [ + { + "$ref": "/wiki/JsonSchema:Manufacturer?action=raw&slot=main" + } + ] + }, + "data_sheet": { + "$ref": "/wiki/JsonSchema:DataSheet?action=raw&slot=main" + }, + "safety_data_sheet": { + "$ref": "/wiki/JsonSchema:SafetyDataSheet?action=raw&slot=main" + }, + "manual": { + "$ref": "/wiki/JsonSchema:Manual?action=raw&slot=main" + }, + "instructions": { + "$ref": "/wiki/JsonSchema:Instructions?action=raw&slot=main" + } + }, + "$defs": { + "generated": { + "$comment": "Autogenerated section - do not edit. Generated from Category:Category Category:OSWffe74f291d354037b318c422591c5023", + "allOf": [ + { + "$ref": "/wiki/Category:Item?action=raw&slot=jsonschema" + }, + { + "$ref": "/wiki/Category:OSW42d82cfc0c304ca48b21e72557a5472f?action=raw&slot=jsonschema" + } + ], + "type": "object", + "uuid": "37a57741-ae2e-4dd4-b29b-1172b6848be8", + "title": "GeneratedPhysicalItem", + "title*": { + "en": "Physical Item", + "de": "Physischer Gegenstand" + }, + "description": "This is an autogenerated partial class definition of 'PhysicalItem'", + "description*": { + "en": "Physical item as opposed to intangible item. It can posses properties such as position, geometry, weight, volume , etc. ", + "de": "Physischer Gegenstand im Gegensatz zu immateriellen Gegenständen. Er kann Eigenschaften wie Position, Geometrie, Gewicht, Volumen usw. besitzen. " + }, + "required": [ + "type" + ], + "properties": { + "type": { + "default": [ + "Category:OSW37a57741ae2e4dd4b29b1172b6848be8" + ] + } + }, + "defaultProperties": [ + "type" + ] + } + } + }, + "revision": { + "revid": 129378, + "parentid": 113874, + "user": "Lukas Gold", + "timestamp": "2026-07-20T10:38:01Z" + } + }, + "Category:OSWca99a9bb9ad24386b222d18a73c2af5e": { + "jsondata": { + "subclass_of": [ + "Category:Category" + ], + "type": [ + "Category:Category" + ], + "uuid": "ca99a9bb-9ad2-4386-b222-d18a73c2af5e", + "label": [ + { + "text": "Physical Item Type", + "lang": "en" + } + ], + "description": [ + { + "text": "Helps to create subcategories of PhysicalItem", + "lang": "en" + } + ], + "name": "PhysicalItemType" + }, + "jsonschema": { + "@context": [ + "/wiki/Category:Category?action=raw&slot=jsonschema", + { + "constituents": { + "@type": "@id", + "@id": "Property:HasStatement", + "@context": { + "property": "Property:HasProperty", + "type": "Property:HasValue" + } + }, + "data_sheet": { + "@id": "Property:HasDataSheet", + "@type": "@id" + }, + "instructions": { + "@id": "Property:HasInstructions", + "@type": "@id" + }, + "manual": { + "@id": "Property:HasManual", + "@type": "@id" + }, + "manufacturer": { + "@id": "Property:HasManufacturer", + "@type": "@id" + }, + "safety_data_sheet": { + "@id": "Property:HasSafetyDataSheet", + "@type": "@id" + }, + "type_no": "Property:HasTypeNumber" + } + ], + "allOf": [ + { + "$ref": "/wiki/Category:Category?action=raw&slot=jsonschema" + } + ], + "type": "object", + "uuid": "ca99a9bb-9ad2-4386-b222-d18a73c2af5e", + "title": "PhysicalItemType", + "title*": { + "en": "Physical Item Type" + }, + "description": "Helps to create subcategories of PhysicalItem", + "description*": { + "en": "Helps to create subcategories of PhysicalItem" + }, + "required": [ + "type" + ], + "defaultPropertis": [ + "type_no", + "manufacturer", + "data_sheet", + "safety_data_sheet", + "supply", + "specifications" + ], + "properties": { + "type": { + "default": [ + "Category:OSWca99a9bb9ad24386b222d18a73c2af5e" + ] + }, + "type_no": { + "$ref": "/wiki/JsonSchema:TypeNumber?action=raw&slot=main" + }, + "manufacturer": { + "$ref": "/wiki/JsonSchema:Manufacturer?action=raw&slot=main" + }, + "data_sheet": { + "$ref": "/wiki/JsonSchema:DataSheet?action=raw&slot=main" + }, + "safety_data_sheet": { + "$ref": "/wiki/JsonSchema:SafetyDataSheet?action=raw&slot=main" + }, + "instructions": { + "$ref": "/wiki/JsonSchema:Instructions?action=raw&slot=main" + }, + "manual": { + "$ref": "/wiki/JsonSchema:Manual?action=raw&slot=main" + }, + "supply": { + "title": "Supply", + "title*": { + "de": "Bezugsmöglichkeit" + }, + "description": "Supply of the material with supplier, article no. and link", + "description*": { + "de": "Bezugsmöglichkeit des Materials mit Lieferant, Artikelnummer und Link" + }, + "type": "array", + "format": "table", + "items": { + "title": "Supply", + "title*": { + "de": "Bezugsmöglichkeit" + }, + "type": "object", + "properties": { + "supplier": { + "title": "Supplier", + "title*": { + "de": "Lieferant" + }, + "type": "string", + "format": "autocomplete", + "options": { + "autocomplete": { + "category": "Category:OSW3cb8cef2225e403092f098f99bc4c472" + } + } + }, + "article_no": { + "title": "Article no.", + "title*": { + "de": "Artikelnummer" + }, + "description": "Article number in the catalogue of the supplier", + "description*": { + "de": "Artikelnummer im Katalog des Lieferanten" + }, + "type": "string" + }, + "comment": { + "title": "Comment", + "title*": { + "de": "Kommentar" + }, + "description": "Comment on the supply of the material", + "description*": { + "de": "Kommentar bezüglich der Bezugsmöglichkeit des Materials" + }, + "type": "string", + "format": "" + }, + "link": { + "title": "Link", + "title*": { + "de": "Link" + }, + "description": "Link to the article in the online catalogue of the supplier", + "description*": { + "de": "Link zum Artikel im Online-Katalog des Lieferanten" + }, + "type": "string", + "format": "uri" + } + } + } + }, + "specifications": { + "title": "Specification of technical properties", + "title*": { + "de": "Spezifikation technischer Eigenschaften" + }, + "type": "array", + "format": "table", + "items": { + "title": "Attribute", + "title*": { + "de": "Attribut" + }, + "type": "object", + "headerTemplate": "{{i1}} - {{self.name}}", + "required": [ + "attribute_name", + "quantity_prop" + ], + "properties": { + "attribute_name": { + "title": "Name", + "type": "array", + "format": "table", + "items": { + "$ref": "/wiki/JsonSchema:Label?action=raw" + }, + "minItems": 1 + }, + "quantity_prop": { + "$ref": "/wiki/JsonSchema:QuantityProperty?action=raw" + } + } + } + }, + "constituents": { + "type": "array", + "format": "tabs", + "title": "Constituents", + "title*": { + "de": "Bestandteile" + }, + "description": "Components or parts", + "items": { + "type": "object", + "title": "Constituent", + "title*": { + "de": "Bestandteil" + }, + "headerTemplate": "{{i1}} - {{self.name}}", + "required": [ + "name", + "type", + "uuid", + "property" + ], + "properties": { + "uuid": { + "title": "UUID", + "type": "string", + "format": "uuid", + "options": { + "hidden": true + } + }, + "property": { + "type": "string", + "default": "Property:HasPart", + "options": { + "hidden": true + } + }, + "is_array": { + "title": "Is array", + "type": "boolean" + }, + "name": { + "title": "Name", + "type": "string" + }, + "type": { + "title": "Type", + "type": "string", + "format": "autocomplete", + "$comment": "Material", + "range": "Category:OSW31ca9a739cb24079b36824045c0832aa", + "options": { + "autocomplete": { + "$comment": "todo: we need an any-subcategory-of shortcut", + "query": "[[SubClassOf::Category:OSW31ca9a739cb24079b36824045c0832aa]]OR[[SubClassOf.SubClassOf::Category:OSW31ca9a739cb24079b36824045c0832aa]]OR[[SubClassOf.SubClassOf.SubClassOf::Category:OSW31ca9a739cb24079b36824045c0832aa]]OR[[SubClassOf.SubClassOf.SubClassOf.SubClassOf::Category:OSW31ca9a739cb24079b36824045c0832aa]]OR[[SubClassOf.SubClassOf.SubClassOf.SubClassOf.SubClassOf::Category:OSW31ca9a739cb24079b36824045c0832aa]]" + } + } + }, + "default": { + "title": "Default value", + "type": "string" + } + } + } + } + } + }, + "revision": { + "revid": 122162, + "parentid": 122150, + "user": "Simon Stier", + "timestamp": "2026-04-04T14:42:51Z" + } + } +} diff --git a/tests/fixtures/category_template_selection/meta/Category.jsondata.json b/tests/fixtures/category_template_selection/meta/Category.jsondata.json new file mode 100644 index 00000000..6743b1ab --- /dev/null +++ b/tests/fixtures/category_template_selection/meta/Category.jsondata.json @@ -0,0 +1,19 @@ +{ + "label": [ + { + "text": "Category (Class)", + "lang": "en" + } + ], + "uuid": "89aafe6d-ae5a-4f29-97ff-df7736d4cab6", + "description": [ + { + "text": "Meta-Category for Categories", + "lang": "en" + } + ], + "name": "CategoryClass", + "type": [ + "Category:Category" + ] +} diff --git a/tests/fixtures/category_template_selection/meta/Category.jsonschema.json b/tests/fixtures/category_template_selection/meta/Category.jsonschema.json new file mode 100644 index 00000000..87dce7cf --- /dev/null +++ b/tests/fixtures/category_template_selection/meta/Category.jsonschema.json @@ -0,0 +1,115 @@ +{ + "@context": [ + { + "exact_ontology_match***": { + "@id": "owl:equivalentClass", + "@type": "@id" + }, + "subclass_of": { + "@id": "rdfs:subClassOf", + "@type": "@id" + }, + "subclass_of*": { + "@id": "Property:SubClassOf", + "@type": "@id" + }, + "version": "owl:versionInfo", + "version*": "Property:HasVersion", + "utf8_icon": "Property:HasUtf8Icon" + }, + "/wiki/Category:OSWffe74f291d354037b318c422591c5023?action=raw&slot=jsonschema" + ], + "allOf": [ + { + "$ref": "/wiki/Category:OSWffe74f291d354037b318c422591c5023?action=raw&slot=jsonschema" + } + ], + "uuid": "89aafe6d-ae5a-4f29-97ff-df7736d4cab6", + "type": "object", + "title": "Category", + "title*": { + "de": "Kategorie (Klasse)", + "en": "Category (Class)" + }, + "required": [ + "subclass_of", + "type" + ], + "defaultProperties": [ + "version" + ], + "properties": { + "type": { + "items": { + "subclassof_range": "Category:Category", + "format": "autocomplete", + "options": { + "autocomplete": { + "$comment": "All categories here since subclassof_range is limited by depth 5", + "query": "[[:Category:%2B]]" + } + } + }, + "default": [ + "Category:Category" + ], + "options": { + "hidden": true + } + }, + "version": { + "title": "Version", + "title*": { + "de": "Version" + }, + "description": "Human-readable version string (e.g. 1.0.1) following https://semver.org/", + "description*": { + "de": "Für Menschen lesbare Versionszeichenfolge (z.B. 1.0.1) gemäß https://semver.org/" + }, + "type": "string", + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + "propertyOrder": -1010, + "default": "0.1.0" + }, + "subclass_of": { + "title": "Supercategories", + "title*": { + "de": "Überkategorien" + }, + "type": "array", + "propertyOrder": -1009, + "items": { + "title": "Category", + "title*": { + "de": "Kategorie" + }, + "type": "string", + "format": "autocomplete", + "subclassof_range": "Category:Entity" + }, + "default": [ + "Category:Entity" + ], + "options": { + "hidden": false + } + }, + "utf8_icon": { + "type": "array", + "title": "UTF8-Icons", + "title*": { + "de": "UTF8-Symbole" + }, + "description": "see https://www.unicode.org/cldr/charts/latest/annotations/romance.html", + "description*": { + "de": "siehe https://www.unicode.org/cldr/charts/latest/annotations/germanic.html" + }, + "items": { + "type": "string", + "title": "UTF8-Icon", + "minLength": 1, + "maxLength": 2 + } + } + } +} diff --git a/tests/fixtures/category_template_selection/meta/Category.schema_template.txt b/tests/fixtures/category_template_selection/meta/Category.schema_template.txt new file mode 100644 index 00000000..83b7159c --- /dev/null +++ b/tests/fixtures/category_template_selection/meta/Category.schema_template.txt @@ -0,0 +1,37 @@ +{ + "@context": [ + {{#each subclass_of}} + "/wiki/{{{.}}}?action=raw\u0026slot=jsonschema"{{#unless @last}},{{/unless}} + {{/each}} + ], + "allOf": [ + {{#if subclass_of}}{{#each subclass_of}} + {"$ref": "/wiki/{{{.}}}?action=raw\u0026slot=jsonschema"}{{#unless @last}},{{/unless}} + {{/each}}{{else}} + {} + {{/if}} + ], + "type": "object", + "uuid": "{{{uuid}}}", + {{#if version}}"x-oold-version": "{{{version}}}",{{/if}} + "title": "{{{name}}}", + "title*": { + {{#each label}} + "{{{lang}}}": "{{{text}}}"{{#unless @last}},{{/unless}} + {{/each}} + }, + "description": "{{{description.[0].text}}}", + "description*": { + {{#each description}} + "{{{lang}}}": "{{{text}}}"{{#unless @last}},{{/unless}} + {{/each}} + }, + "required": [ + "type" + ], + "properties": { + "type": { + "default": ["{{{_page_title}}}"] + } + } +} diff --git a/tests/fixtures/category_template_selection/meta/CharacteristicType.jsondata.json b/tests/fixtures/category_template_selection/meta/CharacteristicType.jsondata.json new file mode 100644 index 00000000..56e3a2eb --- /dev/null +++ b/tests/fixtures/category_template_selection/meta/CharacteristicType.jsondata.json @@ -0,0 +1,30 @@ +{ + "subclass_of": [ + "Category:Entity" + ], + "type": [ + "Category:Category" + ], + "uuid": "ffe74f29-1d35-4037-b318-c422591c5023", + "label": [ + { + "text": "Characteristic Type", + "lang": "en" + }, + { + "text": "Charakteristik-Typ", + "lang": "de" + } + ], + "description": [ + { + "text": "A set of properties intended to be used in other characteristic types or classes", + "lang": "en" + }, + { + "text": "Ein Satz von Eigenschaften zur Nutzung in anderen Charakteristik-Typen oder Klassen", + "lang": "de" + } + ], + "name": "CharacteristicType" +} diff --git a/tests/fixtures/category_template_selection/meta/CharacteristicType.jsonschema.json b/tests/fixtures/category_template_selection/meta/CharacteristicType.jsonschema.json new file mode 100644 index 00000000..bb2110fb --- /dev/null +++ b/tests/fixtures/category_template_selection/meta/CharacteristicType.jsonschema.json @@ -0,0 +1,993 @@ +{ + "@context": [ + { + "subclass_of": { + "@id": "rdfs:subClassOf", + "@type": "@id" + }, + "subclass_of*": { + "@id": "Property:SubClassOf", + "@type": "@id" + }, + "characteristics": { + "@id": "Property:HasCharacteristic", + "@type": "@id" + }, + "characteristic": { + "@id": "Property:HasCharacteristic", + "@type": "@id" + }, + "properties": { + "@id": "Property:HasProperty", + "@type": "@id" + }, + "metaclass": { + "@id": "Property:HasMetaCategory", + "@type": "@id" + } + }, + "/wiki/Category:Entity?action=raw&slot=jsonschema" + ], + "allOf": [ + { + "$ref": "/wiki/Category:Entity?action=raw&slot=jsonschema" + } + ], + "type": "object", + "uuid": "ffe74f29-1d35-4037-b318-c422591c5023", + "title": "CharacteristicType", + "title*": { + "en": "Characteristic Type", + "de": "Charakteristik-Typ" + }, + "description": "A set of properties intended to be used in other characteristic types or classes", + "description*": { + "en": "A set of properties intended to be used in other characteristic types or classes", + "de": "Ein Satz von Eigenschaften zur Nutzung in anderen Charakteristik-Typen oder Klassen" + }, + "required": [ + "type", + "subclass_of", + "label" + ], + "defaultProperties": [ + "characteristics", + "properties" + ], + "properties": { + "type": { + "items": { + "format": "autocomplete", + "subclassof_range": "Category:OSWffe74f291d354037b318c422591c5023", + "$comment": "Explicit query because subclassof_range does not include the specified category itself. Includes 5 levels of SubClassOf depth.", + "options": { + "autocomplete": { + "query": "[[:Category:OSWffe74f291d354037b318c422591c5023]] OR [[SubClassOf::Category:OSWffe74f291d354037b318c422591c5023]] OR [[SubClassOf.SubClassOf::Category:OSWffe74f291d354037b318c422591c5023]] OR [[SubClassOf.SubClassOf.SubClassOf::Category:OSWffe74f291d354037b318c422591c5023]] OR [[SubClassOf.SubClassOf.SubClassOf.SubClassOf::Category:OSWffe74f291d354037b318c422591c5023]] OR [[SubClassOf.SubClassOf.SubClassOf.SubClassOf.SubClassOf::Category:OSWffe74f291d354037b318c422591c5023]]" + } + } + }, + "default": [ + "Category:OSWffe74f291d354037b318c422591c5023" + ], + "options": { + "collapsed": true + } + }, + "subclass_of": { + "title": "Supercategories (is a)", + "title*": { + "de": "Überkategorien (ist eine)" + }, + "description": "Parent class(es)", + "description*": { + "de": "Elternklasse(n)" + }, + "options": { + "$comment": "should be an $example", + "infoText": "A car is a vehicle" + }, + "type": "array", + "format": "table", + "propertyOrder": -1009, + "items": { + "title": "Category", + "title*": { + "en": "Category", + "de": "Kategorie" + }, + "type": "string", + "format": "autocomplete", + "$comment": "range MetaCharacteristic, subclass_of range Characteristic", + "range": "Category:OSWffe74f291d354037b318c422591c5023", + "subclassof_range": "Category:OSW93ccae36243542ceac6c951450a81d47" + }, + "default": [ + "Category:OSW93ccae36243542ceac6c951450a81d47" + ], + "minItems": 1 + }, + "label": { + "propertyOrder": -1008 + }, + "metaclass": { + "title": "Metaclasses/-categories", + "title*": { + "de": "Metaklassen/-kategorien" + }, + "type": "array", + "format": "table", + "propertyOrder": -1007, + "items": { + "title": "Metaclass", + "title*": { + "de": "Metaklasse" + }, + "type": "string", + "format": "autocomplete", + "subclassof_range": "Category:Category" + } + }, + "characteristics": { + "title": "Characteristics (mixin)", + "title*": { + "de": "Charakteristiken (mixin)" + }, + "description": "Include all properties of an existing characteristic without subclassof-relation", + "description*": { + "de": "Wiederverwendete Schema-Bestandteile ohne starke Elternklassenbeziehung" + }, + "options": { + "$comment": "should be an $example", + "infoText": "A human has a physical dimension without beeing a physical dimension" + }, + "$example": "", + "type": "array", + "propertyOrder": -1009, + "items": { + "title": "Characteristic", + "title*": { + "de": "Charakteristik" + }, + "type": "string", + "format": "autocomplete", + "$comment": "range MetaCharacteristic, subclass_of range Characteristic", + "range": "Category:OSWffe74f291d354037b318c422591c5023", + "subclassof_range": "Category:OSW93ccae36243542ceac6c951450a81d47" + } + }, + "properties": { + "title": "Own properties", + "title*": { + "de": "Eigene Attribute" + }, + "description": "Properties that are defined in addition to the ones inherited by superclasses and imported characteristics", + "description*": { + "de": "Attribute die zusätzlich zu den durch Elternklassen und importierte Charakteristiken geerbten Attributen definiert werden" + }, + "propertyOrder": -1009, + "type": "array", + "format": "tabs", + "options": { + "enable_array_copy": true, + "array_copy_ignore": [ + "uuid" + ] + }, + "Xeval_template": [ + { + "type": "mustache-wikitext", + "mode": "render", + "value": "{{=<% %>=}} <%#properties%>* <%name%> {{#if: <%rdf_property%> |({{Viewer/Link |page=<%rdf_property%>}}) }}: {{#if: <%characteristic%> |{{Viewer/Link |page=<%characteristic%>}} }} <br> <%/properties%>" + } + ], + "items": { + "id": "own_property", + "headerTemplate": "{{ i1 }} - {{ self.name }}", + "required": [ + "uuid", + "name" + ], + "defaultProperties": [ + "rdf_property", + "label", + "description", + "required", + "recommended", + "array" + ], + "options": { + "copy_ignore": [ + "uuid" + ] + }, + "properties": { + "uuid": { + "title": "UUID", + "type": "string", + "$comment": "format uuid field are copied, if not set in copy_ignore", + "format": null, + "dynamic_template": "{{_uuid_}}", + "options": { + "hidden": true, + "dynamic_template": { + "override": "empty" + } + } + }, + "name": { + "type": "string", + "pattern": "^[a-z0-9]+(_[a-z0-9]+)*$", + "title": "Name", + "description": "Technical name / keyword. Should be lowercase without spaces and special chars except underscore", + "description*": { + "de": "Technischer Name / Schlüsselwort. Sollte in Kleinbuchstaben ohne Leerzeichen und Sonderzeichen außer Unterstrich sein." + } + }, + "label": { + "type": "array", + "title": "Label(s)", + "title*": { + "de": "Bezeichnung(en)" + }, + "description": "At least one label is required.", + "description*": { + "de": "Mindestens eine Bezeichnung ist erforderlich." + }, + "format": "table", + "eval_template": [ + { + "type": "mustache-wikitext", + "mode": "render", + "$comment": "Displays value according to user language with eng as fallback option. Note: {{=<% %>=}} changes mustache expression from {{..}} to <% %> for mixing with wikitext templates", + "value": "{{=<% %>=}} {{#switch:{{USERLANGUAGECODE}} <%#label%> | {{#ifeq: <%lang%>|en|#default|<%lang%>}} = <%text%> <%/label%> }}" + } + ], + "items": { + "$ref": "/wiki/JsonSchema:Label?action=raw" + }, + "minItems": 1, + "options": { + "grid_columns": 12 + } + }, + "description": { + "title": "Description", + "title*": { + "de": "Beschreibung" + }, + "type": "array", + "format": "table", + "eval_template": { + "type": "mustache-wikitext", + "mode": "render", + "value": "{{=<% %>=}} {{#switch:{{USERLANGUAGECODE}} <%={{ }}=%> {{#description}} |{{lang}}={{text}} {{/description}} {{=<% %>=}} }}" + }, + "items": { + "$ref": "/wiki/JsonSchema:Description?action=raw" + }, + "options": { + "grid_columns": 12 + } + }, + "rdf_property": { + "type": "string", + "title": "Internal vocabulary mapping", + "title*": { + "de": "Interne Vokabular-Zuordnung" + }, + "description": "Maps a property keyword to an internal RDF property as defined in JSON-LD. Required to query this information, e.g., for tabular or graph views. Also adopts value range and data type restrictions of the property if defined.", + "description*": { + "de": "Ordnet gemäß JSON-LD einem Eigenschafts-Schlüsselwort ein internes RDF-Property zu. Wird benötigt um diese Information abzufragen, etwa für tabellarische oder Graph-Ansichten. Übernimmt auch die Wertebereichs- und Datentypbeschränkungen des Propertys, falls definiert." + }, + "format": "autocomplete", + "range": "Category:Property", + "watch": { + "label": "own_property.label", + "description": "own_property.description", + "range": "own_property.range" + }, + "options": { + "autocomplete": { + "query": "[[Category:Property]]|?HasLabel=label|?HasDescription=description|?HasRange=range", + "field_maps": [ + { + "source_path": "$", + "template": "[{{#each result.printouts.label}}{\"text\": \"{{{.}}}\"}{{/each}}]", + "target_path": "$(label)" + }, + { + "source_path": "$", + "$comment": "multilang properties are normalized", + "template_multilang": "[{{#each result.printouts.description}}{\"text\": \"{{{Text.item.[0]}}}\", \"lang\": \"{{{[Language code].item.[0]}}}\"}{{/each}}]", + "template": "[{{#each result.printouts.description}}{\"text\": \"{{{.}}}\"}{{/each}}]", + "target_path": "$(description)" + }, + { + "__disabled__source_path": "$..range..fulltext", + "source_path": "$", + "template": "[\"{{{result.printouts.range.[0].fulltext}}}\"]", + "target_path": "$(range)" + } + ] + } + } + }, + "external_rdf_properties": { + "title": "External Vocabulary mapping", + "title*": { + "de": "Externe Vokabular-Zuordnung" + }, + "description": "Maps a property keyword to an external RDF property as defined in JSON-LD. Required to exchange data with other systems using those properties in its vocabulary.", + "description*": { + "de": "Ordnet gemäß JSON-LD einem Eigenschafts-Schlüsselwort ein RDF-Property zu. Erforderlich für den Datenaustausch mit anderen Systemen, die diese Properties in ihrem Vokabular verwenden." + }, + "type": "array", + "items": { + "$comment": "ToDo: Create context notation with '*'. ToDo: Should validate IRIs (https://schema.org/xxx, schema:xxx, Property:xxx)", + "type": "string", + "options": { + "compact": true + } + } + }, + "required": { + "type": "boolean", + "format": "checkbox", + "title": "Required", + "title*": { + "de": "Verpflichtend" + }, + "description": "Indicates whether this field is mandatory. Mandatory properties are always displayed in the editor.", + "description*": { + "de": "Gibt an, ob dieses Attribut erforderlich ist." + }, + "default": false + }, + "recommended": { + "type": "boolean", + "format": "checkbox", + "title": "Recommended", + "title*": { + "de": "Empfohlen" + }, + "description": "Indicates whether this field is suggested but not required. If properties are recommended they are displayed by default. Otherwise they must be selected explicitly by the user", + "description*": { + "de": "Gibt an, ob dieses Attribut vorgeschlagen wird aber nicht verpflichend ist. Wenn Attribute empfohlen sind werden sie standardmäßig angezeigt. Andernfalls müssen sie vom Nutzer explizit ausgewählt werden." + }, + "default": true + }, + "array": { + "type": "boolean", + "format": "checkbox", + "title": "Array", + "title*": { + "de": "Liste" + }, + "description": "Indicates whether this property can contain multiple values.", + "description*": { + "de": "Gibt an, ob dieses Attribut mehrere Werte enthalten kann." + }, + "default": false + } + }, + "oneOf": [ + { + "title": "NumberProperty", + "title*": { + "de": "Zahlenwert-Attribut" + }, + "description": "Attribute for numerical values (integers or decimals)", + "description*": { + "de": "Attribut für Zahlenwerte (ganze Zahlen oder Dezimalzahlen)" + }, + "required": [ + "type" + ], + "defaultProperties": [], + "properties": { + "type": { + "title": "Number Type", + "title*": { + "de": "Zahlentyp" + }, + "description": "Type of number: whole numbers or decimals", + "description*": { + "de": "Zahlentyp: ganze Zahlen oder Dezimalzahlen" + }, + "type": "string", + "enum": [ + "number", + "integer" + ], + "default": "number", + "options": { + "enum_titles": [ + "Decimals (number)", + "Whole numbers (integer)" + ], + "enum_titles*": { + "de": [ + "Dezimalzahlen (number)", + "Ganze Zahlen (integer)" + ] + } + } + }, + "minimum": { + "title": "Minimum", + "title*": { + "de": "Minimum" + }, + "description": "Minimum allowed value (inclusive)", + "description*": { + "de": "Minimal erlaubter Wert (einschließlich)" + }, + "type": "number" + }, + "maximum": { + "title": "Maximum", + "title*": { + "de": "Maximum" + }, + "description": "Maximum allowed value (inclusive)", + "description*": { + "de": "Maximal erlaubter Wert (einschließlich)" + }, + "type": "number" + }, + "exclusive_minimum": { + "title": "Exclusive minimum", + "title*": { + "de": "Exklusives Minimum" + }, + "description": "Value must be strictly greater than this", + "description*": { + "de": "Wert muss strikt größer als dieser sein" + }, + "type": "number" + }, + "exclusive_maximum": { + "title": "Exclusive maximum", + "title*": { + "de": "Exklusives Maximum" + }, + "description": "Value must be strictly less than this", + "description*": { + "de": "Wert muss strikt kleiner als dieser sein" + }, + "type": "number" + }, + "multiple_of": { + "title": "Multiple of", + "title*": { + "de": "Vielfaches von" + }, + "description": "Value must be a multiple of this number (e.g., step size)", + "description*": { + "de": "Wert muss ein Vielfaches dieser Zahl sein (z. B. Schrittweite)" + }, + "type": "number", + "exclusiveMinimum": 0 + }, + "enum": { + "title": "Allowed values", + "title*": { + "de": "Erlaubte Werte" + }, + "description": "Predefined list of allowed numerical values", + "description*": { + "de": "Vordefinierte Liste erlaubter Zahlenwerte" + }, + "type": "array", + "items": { + "type": "number" + } + }, + "default": { + "title": "Default value", + "title*": { + "de": "Standardwert" + }, + "description": "Default value for this property", + "description*": { + "de": "Standardwert für dieses Attribut" + }, + "type": "number" + } + } + }, + { + "title": "TextProperty", + "title*": { + "de": "Text-Attribut" + }, + "description": "Attribute for textual values", + "description*": { + "de": "Attribut für Textwerte" + }, + "required": [ + "type" + ], + "defaultProperties": [], + "properties": { + "type": { + "type": "string", + "const": "string", + "default": "string", + "options": { + "hidden": "true" + } + }, + "format": { + "title": "Format", + "title*": { + "de": "Format" + }, + "description": "Display and validation format for string values", + "description*": { + "de": "Anzeige- und Validierungsformat für Textwerte" + }, + "type": "string", + "enum": [ + "date", + "date-time", + "time", + "email", + "uri", + "color" + ], + "options": { + "enum_titles": [ + "Date (date)", + "Date and time (date-time)", + "Time (time)", + "Email (email)", + "URI (uri)", + "Color (color)" + ], + "enum_titles*": { + "de": [ + "Datum (date)", + "Datum und Uhrzeit (date-time)", + "Uhrzeit (time)", + "E-Mail (email)", + "URI (uri)", + "Farbe (color)" + ] + } + } + }, + "pattern": { + "title": "Regular expression", + "title*": { + "de": "Regulärer Ausdruck" + }, + "description": "A regular expression pattern the value must match", + "description*": { + "de": "Ein regulärer Ausdruck, dem der Wert entsprechen muss" + }, + "type": "string" + }, + "min_length": { + "title": "Minimum length", + "title*": { + "de": "Mindestlänge" + }, + "description": "Minimum number of characters", + "description*": { + "de": "Mindestanzahl von Zeichen" + }, + "type": "integer", + "minimum": 0 + }, + "max_length": { + "title": "Maximum length", + "title*": { + "de": "Maximallänge" + }, + "description": "Maximum number of characters", + "description*": { + "de": "Maximale Anzahl von Zeichen" + }, + "type": "integer", + "minimum": 0 + }, + "enum": { + "title": "Allowed values", + "title*": { + "de": "Erlaubte Werte" + }, + "description": "Predefined list of allowed values", + "description*": { + "de": "Vordefinierte Liste erlaubter Werte" + }, + "type": "array", + "items": { + "type": "string" + } + }, + "default": { + "title": "Default value", + "title*": { + "de": "Standardwert" + }, + "description": "Default value for this property", + "description*": { + "de": "Standardwert für dieses Attribut" + }, + "type": "string" + } + } + }, + { + "title": "BooleanProperty", + "title*": { + "de": "Wahrheitswert-Attribut" + }, + "description": "Attribute for true/false values", + "description*": { + "de": "Attribut für Wahrheitswerte (wahr/falsch)" + }, + "required": [ + "type" + ], + "defaultProperties": [], + "properties": { + "type": { + "type": "string", + "const": "boolean", + "default": "boolean", + "options": { + "hidden": "true" + } + }, + "default": { + "title": "Default value", + "title*": { + "de": "Standardwert" + }, + "description": "Default value for this property", + "description*": { + "de": "Standardwert für dieses Attribut" + }, + "type": "boolean", + "format": "checkbox" + } + } + }, + { + "title": "LinkProperty", + "title*": { + "de": "Verweis-Attribut" + }, + "description": "Attribute that links to another entity via autocomplete", + "description*": { + "de": "Attribut, das über Autovervollständigung auf eine andere Entität verweist" + }, + "required": [ + "type" + ], + "defaultProperties": [ + "range" + ], + "properties": { + "type": { + "type": "string", + "const": "iri", + "default": "iri", + "options": { + "hidden": "true" + } + }, + "range": { + "title": "Autocomplete value range", + "title*": { + "de": "Wertebereich für Auto-Vervollständigung" + }, + "description": "Enables autocomplete support by looking up all elements that are in at least one of the listed categories. Currently only a single category is supported.", + "description*": { + "de": "Ermöglicht die Unterstützung der automatischen Vervollständigung durch Nachschlagen aller Elemente, die mindestens in einer der aufgeführten Kategorien enthalten sind. Derzeit wird nur eine einzelne Kategorie unterstützt." + }, + "type": "array", + "maxItems": 1, + "items": { + "type": "string", + "format": "autocomplete", + "range": "Category:Category", + "subclassof_range": "Category:Item", + "options": { + "autocomplete": { + "query": "[[:Category:%2B]]" + } + } + } + }, + "subclassof_range": { + "title": "Subclass value range", + "title*": { + "de": "Wertebereich für Unterklassen" + }, + "description": "Offer the subclasses of this class instead of instances.", + "description*": { + "de": "Bietet die Unterklassen dieser Klasse an statt Instanzen." + }, + "type": "string", + "format": "autocomplete", + "range": "Category:Category" + }, + "query": { + "title": "Custom autocomplete query", + "title*": { + "de": "Eigene Abfrage für die Auto-Vervollständigung" + }, + "description": "Semantic MediaWiki query that replaces the generated one. Use only when range and subclass value range cannot express the constraint.", + "description*": { + "de": "Semantic-MediaWiki-Abfrage, die die erzeugte Abfrage ersetzt. Nur verwenden, wenn Wertebereich und Unterklassen-Wertebereich nicht ausreichen." + }, + "$comment": "This will allow complex expressions like '(A & B) | C' in the future", + "type": "string" + } + } + }, + { + "title": "ComplexProperty", + "title*": { + "en": "Complex property", + "de": "Komplexes Attribut" + }, + "description": "Attribute that contains a subobject, defined by another characteristic type or class", + "description*": { + "de": "Attribut, das auf ein Subobjekt verweist, das durch eine andere Charakteristik oder Klasse definiert ist" + }, + "required": [ + "type", + "characteristic" + ], + "properties": { + "type": { + "type": "string", + "const": "object", + "default": "object", + "options": { + "hidden": "true" + } + }, + "characteristic": { + "title": "Characteristic", + "title*": { + "de": "Charakteristik" + }, + "description": "The characteristic or class defining the object this property points to", + "description*": { + "de": "Die Charakteristik oder Klasse die das Objekt spezifiziert auf die das Attribut zeigt" + }, + "type": "string", + "format": "autocomplete", + "$comment": "range MetaCharacteristic, subclass_of range Characteristic for convenience. Should allow also classes in general.", + "range": "Category:OSWffe74f291d354037b318c422591c5023", + "subclassof_range": "Category:OSW93ccae36243542ceac6c951450a81d47", + "#range": "Category:Category", + "#subclassof_range": "Category:Item" + }, + "rdf_property": { + "range": "Category:ObjectProperty", + "default": "Property:HasCharacteristic" + }, + "default_unit": { + "title": "Default unit override", + "title*": { + "de": "Standard-Einheit (Überschreibung)" + }, + "description": "Override the default display unit of the referenced characteristic (only relevant for quantity-based characteristics)", + "description*": { + "de": "Überschreibt die Standard-Anzeigeeinheit der referenzierten Charakteristik (nur relevant für mengenbasierte Charakteristiken)" + }, + "type": "string", + "format": "autocomplete", + "watch": { + "characteristic_w": "own_property.characteristic" + }, + "$comment": "Query finds units via the characteristic's SubClassOf chain to the fundamental quantity. Returns empty for non-quantity characteristics (self-guarding).", + "options": { + "autocomplete": { + "query": "[[-HasUnit.-HasQuantity::{{$(characteristic_w)}}]][[HasSymbol::like:*{{_user_input}}*]]OR[[-HasUnit.-HasQuantity.-SubClassOf::{{$(characteristic_w)}}]][[HasSymbol::like:*{{_user_input}}*]]OR[[-HasUnit.-HasQuantity.-SubClassOf.-SubClassOf::{{$(characteristic_w)}}]][[HasSymbol::like:*{{_user_input}}*]]OR[[-HasUnit.-HasQuantity.-SubClassOf.-SubClassOf.-SubClassOf::{{$(characteristic_w)}}]][[HasSymbol::like:*{{_user_input}}*]]|?HasSymbol=label" + } + } + } + } + }, + { + "title": "TypedInstanceProperty", + "title*": { + "en": "Typed instance property", + "de": "Typisiertes Instanz-Attribut" + }, + "description": "Attribute where the user first picks a subclass and then an instance of that subclass. Generates the type picker, the instance picker, the narrowing query and the JSON-LD context.", + "description*": { + "de": "Attribut, bei dem zuerst eine Unterklasse und dann eine Instanz dieser Unterklasse gewählt wird. Erzeugt Typ-Auswahl, Instanz-Auswahl, einschränkende Abfrage und JSON-LD-Kontext." + }, + "required": [ + "type", + "instance_property" + ], + "defaultProperties": [ + "base_class", + "instance_property", + "default_type" + ], + "properties": { + "type": { + "type": "string", + "const": "typed_instance", + "default": "typed_instance", + "options": { + "hidden": "true" + } + }, + "base_class": { + "title": "Base class", + "title*": { + "de": "Basisklasse" + }, + "description": "The class whose subclasses the type picker offers. Give either this or \"base class from field\".", + "description*": { + "de": "Die Klasse, deren Unterklassen zur Auswahl stehen." + }, + "type": "string", + "format": "autocomplete", + "range": "Category:Category", + "subclassof_range": "Category:Item" + }, + "instance_property": { + "title": "Instance property", + "title*": { + "de": "Instanz-Attribut" + }, + "description": "The semantic edge between this entity and the selected instance.", + "description*": { + "de": "Die semantische Kante zwischen dieser Entität und der gewählten Instanz." + }, + "type": "string", + "format": "autocomplete", + "range": "Category:ObjectProperty" + }, + "default_type": { + "title": "Default subclass", + "title*": { + "de": "Vorausgewählte Unterklasse" + }, + "description": "Optional. Preselects a subclass and narrows the instance picker before the user chooses.", + "description*": { + "de": "Optional. Wählt eine Unterklasse vor und schränkt die Instanz-Auswahl vorab ein." + }, + "type": "string", + "format": "autocomplete", + "range": "Category:Category" + }, + "subfield_prefix": { + "title": "Subfield name prefix", + "title*": { + "de": "Präfix der Unterfelder" + }, + "description": "Optional. Prefix of the generated subfield names, which are stored in page data. Defaults to the property name. Set it only to keep the names an existing property already uses.", + "description*": { + "de": "Optional. Präfix der erzeugten Unterfeldnamen, die in den Seitendaten gespeichert werden. Standard ist der Attributname. Nur setzen, um vorhandene Namen zu erhalten." + }, + "type": "string", + "pattern": "^[a-z_]+$" + }, + "context_label": { + "title": "Include label and description", + "title*": { + "de": "Bezeichnung und Beschreibung einschließen" + }, + "description": "Optional. Adds label and description subfields, for naming the role the referenced instance plays here.", + "description*": { + "de": "Optional. Fügt Unterfelder für Bezeichnung und Beschreibung hinzu." + }, + "type": "boolean", + "format": "checkbox", + "default": false + }, + "extra_subfields": { + "title": "Extra subfields", + "title*": { + "de": "Zusätzliche Unterfelder" + }, + "description": "Escape hatch. Additional subfields stored inside each entry, beside the generated ones.", + "description*": { + "de": "Notausgang. Zusätzliche Unterfelder, die neben den erzeugten in jedem Eintrag gespeichert werden." + }, + "type": "array", + "format": "table", + "items": { + "type": "object", + "required": [ + "name", + "type" + ], + "properties": { + "name": { + "title": "Name", + "type": "string", + "pattern": "^[a-z_][a-z0-9_]*$" + }, + "title": { + "title": "Label", + "title*": { + "de": "Bezeichnung" + }, + "description": "Optional. Defaults to the name.", + "type": "string" + }, + "type": { + "title": "Type", + "title*": { + "de": "Typ" + }, + "type": "string", + "enum": [ + "string", + "number", + "integer", + "boolean" + ], + "default": "string" + }, + "array": { + "title": "Multiple values", + "title*": { + "de": "Mehrere Werte" + }, + "type": "boolean", + "format": "checkbox", + "default": false + } + } + } + }, + "base_class_from": { + "title": "Base class from field", + "title*": { + "de": "Basisklasse aus Feld" + }, + "description": "Two-level narrowing. Path of the field that holds the base class, in the form <id anchor>.<field name>, for example slot.slot_type_restriction. Use this instead of \"base class\" when the base class is chosen elsewhere in the form rather than fixed here.", + "description*": { + "de": "Zweistufige Einschränkung. Pfad des Feldes, das die Basisklasse enthält, in der Form <id>.<Feldname>. Statt \"Basisklasse\" verwenden, wenn die Basisklasse an anderer Stelle im Formular gewählt wird." + }, + "type": "string", + "pattern": "^[a-z_][a-z0-9_]*\\.[a-z_][a-z0-9_]*$" + }, + "instance_range": { + "title": "Instance value range", + "title*": { + "de": "Wertebereich der Instanz-Auswahl" + }, + "description": "Class the instance picker draws from. Defaults to the default subclass, then to the base class. Required with \"base class from field\", because the base class is then only known at runtime.", + "description*": { + "de": "Klasse, aus der die Instanz-Auswahl schöpft. Standard ist die vorausgewählte Unterklasse, sonst die Basisklasse." + }, + "type": "string", + "format": "autocomplete", + "range": "Category:Category" + }, + "omit_uuid": { + "title": "No uuid subfield", + "title*": { + "de": "Kein uuid-Unterfeld" + }, + "description": "Leave out the generated uuid subfield. Set it only where the existing property has none, so that stored entries stay valid.", + "description*": { + "de": "Lässt das erzeugte uuid-Unterfeld weg. Nur setzen, wenn das vorhandene Attribut keines hat." + }, + "type": "boolean", + "format": "checkbox", + "default": false + } + } + } + ] + } + } + } +} diff --git a/tests/fixtures/category_template_selection/meta/CharacteristicType.schema_template.txt b/tests/fixtures/category_template_selection/meta/CharacteristicType.schema_template.txt new file mode 100644 index 00000000..b434c53e --- /dev/null +++ b/tests/fixtures/category_template_selection/meta/CharacteristicType.schema_template.txt @@ -0,0 +1,143 @@ +{ + "@context": [ + {{#each subclass_of}} + "/wiki/{{{.}}}?action=raw\u0026slot=jsonschema"{{#unless @last}},{{/unless}} + {{/each}} + {{#each characteristics}}, + "/wiki/{{{.}}}?action=raw\u0026slot=jsonschema" + {{/each}}{{#if subclass_of}},{{else}}{{#if characteristics}},{{/if}}{{/if}}{ + {{#join properties}} + {{#if instance_property}}"{{name}}": {"@id": "Property:HasStatement", "@type": "@id", "@context": { + "{{#if subfield_prefix}}{{{subfield_prefix}}}{{else}}{{{name}}}{{/if}}_instance_property": "Property:HasProperty", + "{{#if subfield_prefix}}{{{subfield_prefix}}}{{else}}{{{name}}}{{/if}}_instance": "Property:HasObject" + }}{{else}}{{#if rdf_property}}"{{name}}": "{{rdf_property}}"{{/if}}{{/if}} + {{/join}} + } + ], + "allOf": [ + {{#each subclass_of}} + {"$ref": "/wiki/{{{.}}}?action=raw\u0026slot=jsonschema" }{{#unless @last}},{{/unless}} + {{/each}} + {{#each characteristics}}, + {"$ref": "/wiki/{{{.}}}?action=raw\u0026slot=jsonschema"} + {{/each}} + ], + "type": "object", + "uuid": "{{{uuid}}}", + "title": "{{{name}}}", + "title*": { + {{#each label}} + "{{{lang}}}": "{{{text}}}"{{#unless @last}},{{/unless}} + {{/each}} + }, + "description": "{{{description.[0].text}}}", + "description*": { + {{#each description}} + "{{{lang}}}": "{{{text}}}"{{#unless @last}},{{/unless}} + {{/each}} + }, + "required": [ + "type" + {{#each properties}} + {{#if required}},"{{name}}"{{/if}} + {{/each}} + ], + "defaultProperties": [ + "type" + {{#each properties}} + {{#if recommended}},"{{name}}"{{/if}} + {{/each}} + ], + "properties": { + "type": { + "default": ["{{{_page_title}}}"] + } + {{#each properties}}, + "{{{name}}}": { + {{#if instance_property}} + {{#if array}}"type": "array", "format": "tabs", "items": { + {{/if}}"type": "object", + "title": "{{{name}}}" + {{#if label}}, "title*": { + {{#each label}} + "{{{lang}}}": "{{{text}}}"{{#unless @last}},{{/unless}} + {{/each}} + }{{/if}} + {{#if description}}, "description": "{{{description.[0].text}}}", + "description*": { + {{#each description}} + "{{{lang}}}": "{{{text}}}"{{#unless @last}},{{/unless}} + {{/each}} + }{{/if}}, + "id": "{{{name}}}", + "required": [{{#unless omit_uuid}}"uuid", {{/unless}}"{{#if subfield_prefix}}{{{subfield_prefix}}}{{else}}{{{name}}}{{/if}}_instance_property", "{{#if subfield_prefix}}{{{subfield_prefix}}}{{else}}{{{name}}}{{/if}}_instance"], + "properties": { + {{#unless omit_uuid}}"uuid": { + "title": "UUID", "type": "string", "format": "uuid", + "dynamic_template": "{{{{raw}}}}{{_uuid_}}{{{{/raw}}}}", + "options": {"hidden": true} + },{{/unless}} + {{#if context_label}} + "label": {"title": "Label", "type": "array", "format": "table", + "items": {"$ref": "/wiki/JsonSchema:Label?action=raw"}}, + "description": {"title": "Description", "type": "array", "format": "table", + "items": {"$ref": "/wiki/JsonSchema:Description?action=raw"}}, + {{/if}} + "{{#if subfield_prefix}}{{{subfield_prefix}}}{{else}}{{{name}}}{{/if}}_type": { + "title": "Type", "type": "string", "format": "autocomplete", + {{#if base_class_from}}"range": "Category:Category", + "watch": {"{{{name}}}_base_": "{{{base_class_from}}}"}, + "options": {"autocomplete": {"query": "[[:$({{{name}}}_base_)]]OR[[SubClassOf::$({{{name}}}_base_)]]OR[[SubClassOf.SubClassOf::$({{{name}}}_base_)]]OR[[SubClassOf.SubClassOf.SubClassOf::$({{{name}}}_base_)]]OR[[SubClassOf.SubClassOf.SubClassOf.SubClassOf::$({{{name}}}_base_)]]OR[[SubClassOf.SubClassOf.SubClassOf.SubClassOf.SubClassOf::$({{{name}}}_base_)]]"}} + {{else}}"subclassof_range": "{{{base_class}}}" + {{/if}} + {{#if default_type}}, "default": "{{{default_type}}}"{{/if}} + }, + "{{#if subfield_prefix}}{{{subfield_prefix}}}{{else}}{{{name}}}{{/if}}_instance_property": { + "type": "string", "default": "{{{instance_property}}}", + "options": {"hidden": true} + }, + "{{#if subfield_prefix}}{{{subfield_prefix}}}{{else}}{{{name}}}{{/if}}_instance": { + "title": "Instance", "type": "string", "format": "autocomplete", + "range": "{{#if instance_range}}{{{instance_range}}}{{else}}{{#if default_type}}{{{default_type}}}{{else}}{{{base_class}}}{{/if}}{{/if}}", + "watch": {"{{{name}}}_type_": "{{{name}}}.{{#if subfield_prefix}}{{{subfield_prefix}}}{{else}}{{{name}}}{{/if}}_type"}, + "options": {"autocomplete": {"query": "[[$({{{name}}}_type_)]]|?Display_title_of=label|?HasImage=image|?HasDescription=description"}} + } + {{#each extra_subfields}}, + "{{{name}}}": {{#if array}}{"type": "array", "items": {{/if}}{"title": "{{#if title}}{{{title}}}{{else}}{{{name}}}{{/if}}", "type": "{{{type}}}"}{{#if array}} }{{/if}} + {{/each}} + } + {{#if array}} } {{/if}} + {{else}} + {{#if array}}"type": "array", "items": { + {{/if}} + "title": "{{{name}}}" + {{#if label}}, "title*": { + {{#each label}} + "{{{lang}}}": "{{{text}}}"{{#unless @last}},{{/unless}} + {{/each}} + }{{/if}} + {{#if description}}, "description": "{{{description.[0].text}}}", + "description*": { + {{#each description}} + "{{{lang}}}": "{{{text}}}"{{#unless @last}},{{/unless}} + {{/each}} + }{{/if}} + {{#if characteristic}}{{#if default_unit}}, "allOf": [{"$ref": "/wiki/{{{characteristic}}}?action=raw\u0026slot=jsonschema"}], "properties": {"unit": {"default": "{{{default_unit}}}"}}{{else}}, "$ref": "/wiki/{{{characteristic}}}?action=raw\u0026slot=jsonschema"{{/if}}{{else}}{{#if rdf_property}}, "$ref": "/wiki/{{{rdf_property}}}?action=raw\u0026slot=jsonschema"{{/if}}{{/if}} + {{#if range}}, "type": "string", "format": "autocomplete", "range": "{{{range}}}" + {{else}}{{#if type}}, "type": "{{{type}}}"{{/if}} + {{#if format}}, "format": "{{{format}}}"{{/if}} + {{/if}} + {{#if pattern}}, "pattern": "{{{pattern}}}"{{/if}} + {{#if minimum}}, "minimum": {{minimum}}{{/if}} + {{#if maximum}}, "maximum": {{maximum}}{{/if}} + {{#if exclusive_minimum}}, "exclusiveMinimum": {{exclusive_minimum}}{{/if}} + {{#if exclusive_maximum}}, "exclusiveMaximum": {{exclusive_maximum}}{{/if}} + {{#if multiple_of}}, "multipleOf": {{multiple_of}}{{/if}} + {{#if min_length}}, "minLength": {{min_length}}{{/if}} + {{#if max_length}}, "maxLength": {{max_length}}{{/if}} + {{#if array}} } {{/if}} + {{/if}} + } + {{/each}} + } +} diff --git a/tests/fixtures/category_template_selection/meta/Entity.jsondata.json b/tests/fixtures/category_template_selection/meta/Entity.jsondata.json new file mode 100644 index 00000000..d9e62a20 --- /dev/null +++ b/tests/fixtures/category_template_selection/meta/Entity.jsondata.json @@ -0,0 +1,27 @@ +{ + "uuid": "ce353767-c628-45bd-9d88-d6eb3009aec0", + "label": [ + { + "text": "Entity", + "lang": "en" + }, + { + "text": "Entität", + "lang": "de" + } + ], + "name": "Entity", + "description": [ + { + "text": "This is meant to be the most basic category. Contrary to its subcategories, it is not meant to be instantiated", + "lang": "en" + }, + { + "text": "Dies soll die aller grundlegendste Kategorie sein. Im Gegensatz zu ihren Subkategorien, ist diese nicht dazu gedacht instanziiert zu werden.", + "lang": "de" + } + ], + "type": [ + "Category:Category" + ] +} diff --git a/tests/fixtures/category_template_selection/meta/Entity.jsonschema.json b/tests/fixtures/category_template_selection/meta/Entity.jsonschema.json new file mode 100644 index 00000000..8660918e --- /dev/null +++ b/tests/fixtures/category_template_selection/meta/Entity.jsonschema.json @@ -0,0 +1,728 @@ +{ + "@context": { + "owl": "http://www.w3.org/2002/07/owl#", + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "schema": "https://schema.org/", + "skos": "https://www.w3.org/TR/skos-reference/", + "wiki": "https://wiki-dev.open-semantic-lab.org/id/", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "odrl": "http://www.w3.org/ns/odrl/2/", + "Category": { + "@id": "wiki:Category-3A", + "@prefix": true + }, + "File": { + "@id": "https://wiki-dev.open-semantic-lab.org/wiki/Special:Redirect/file/", + "@prefix": true + }, + "Item": { + "@id": "wiki:Item-3A", + "@prefix": true + }, + "Property": { + "@id": "wiki:Property-3A", + "@prefix": true + }, + "attachments*": { + "@id": "Property:HasFileAttachment", + "@type": "@id" + }, + "based_on": { + "@id": "skos:isBasedOn", + "@type": "@id" + }, + "based_on*": { + "@id": "Property:IsBasedOn", + "@type": "@id" + }, + "close_ontology_match": { + "@id": "skos:closeMatch", + "@type": "@id" + }, + "close_ontology_match*": { + "@id": "Property:Equivalent_URI", + "@type": "@id" + }, + "close_ontology_match**": { + "@id": "Property:HasCloseOntologyMatch", + "@type": "@id" + }, + "description": { + "@id": "skos:definition" + }, + "description*": { + "@id": "Property:HasDescription" + }, + "exact_ontology_match": { + "@id": "skos:exactMatch", + "@type": "@id" + }, + "exact_ontology_match*": { + "@id": "Property:Equivalent_URI", + "@type": "@id" + }, + "exact_ontology_match**": { + "@id": "Property:HasExactOntologyMatch", + "@type": "@id" + }, + "image": { + "@id": "schema:image", + "@type": "@id" + }, + "image*": { + "@id": "Property:HasImage", + "@type": "@id" + }, + "keywords": { + "@id": "schema:keywords", + "@type": "@id" + }, + "keywords*": { + "@id": "Property:HasKeyword", + "@type": "@id" + }, + "label": { + "@id": "skos:prefLabel" + }, + "label*": { + "@id": "Property:HasLabel" + }, + "lang": { + "@id": "@language" + }, + "meta": { + "@id": "Property:HasMeta", + "@type": "@id", + "@context": { + "change_id": { + "@id": "Property:HasChangeId", + "@type": "xsd:string" + } + } + }, + "name*": { + "@id": "Property:HasName" + }, + "classification_categories": { + "@id": "Property:Category", + "@type": "@id" + }, + "classification_categories*": { + "@id": "Property:HasClassificationCategory", + "@type": "@id" + }, + "query_label": { + "@id": "Property:HasLabel", + "@type": "@id" + }, + "rdf_type": { + "@id": "@type", + "@type": "@id" + }, + "short_name": { + "@id": "Property:HasShortName" + }, + "access_policy": { + "@id": "odrl:hasPolicy", + "@type": "@id", + "@context": { + "usergroup_permissions": { + "@id": "odrl:permission", + "@type": "@id", + "@context": { + "assignee_usergroup": { + "@id": "odrl:assignee", + "@type": "@id" + }, + "action": { + "@id": "odrl:action", + "@type": "@id" + } + } + }, + "user_permissions": { + "@id": "odrl:permission", + "@type": "@id", + "@context": { + "assignee_user": { + "@id": "odrl:assignee", + "@type": "@id" + }, + "action": { + "@id": "odrl:action", + "@type": "@id" + } + } + } + } + }, + "statements": { + "@id": "Property:HasStatement", + "@type": "@id" + }, + "text": { + "@id": "@value" + }, + "type": { + "@id": "Property:HasType", + "@type": "@id" + }, + "type*": { + "@id": "Property:HasSchema", + "@type": "@id" + }, + "uuid*": { + "@id": "Property:HasUuid" + } + }, + "uuid": "ce353767-c628-45bd-9d88-d6eb3009aec0", + "title": "Entity", + "type": "object", + "id": "entity", + "required": [ + "uuid", + "label" + ], + "defaultProperties": [ + "description" + ], + "properties": { + "type": { + "title": "Types/Categories", + "title*": { + "de": "Typen/Kategorien" + }, + "propertyOrder": -1010, + "type": "array", + "format": "table", + "eval_template": [ + { + "type": "mustache-wikitext", + "mode": "render", + "value": "{{=<% %>=}} <%#type%> {{Viewer/Link |page=<%.%> }} <br><%/type%>" + } + ], + "items": { + "title": "Type", + "title*": { + "de": "Typ" + }, + "type": "string" + }, + "default": [ + "Category:Entity" + ], + "minItems": 1 + }, + "rdf_type": { + "title": "Additional RDF type(s)", + "title*": { + "de": "Zusätzliche(r) RDF-Typ(en)" + }, + "description": "Declares additional type(s) for this entity, thereby stating that this entity is an instance of an additional RDF class. See also https://schema.org/additionalType. Category:Entity is not directly instantiated. Therefor the inheriting classes (Category:Category, Category:Item, Category:Property) decide on the implementation and mapping within the \"@context\". The value must be a full IRI, e.g. https://schema.org/Person", + "description*": { + "de": "Deklariert zusätzliche(n) Typ(en) für diese Entität und gibt damit an, dass diese Entität eine Instanz einer zusätzlichen RDF-Klasse ist. Siehe auch https://schema.org/additionalType. Category:Entity wird nicht direkt instanziiert. Daher entscheiden die erbenden Klassen (Category:Category, Category:Item, Category:Property) über die Implementierung und das Mapping innerhalb des \"@context\". Der Wert muss eine vollständige IRI sein, z.B. https://schema.org/Person" + }, + "type": "array", + "format": "table", + "eval_template": [ + { + "type": "mustache-wikitext", + "mode": "render", + "value": "{{#rdf_type}} {{=<% %>=}} {{Viewer/Link |url= <%={{ }}=%> {{{.}}} {{=<% %>=}} }} <br><%={{ }}=%>{{/rdf_type}}" + } + ], + "items": { + "title": "RDF type", + "title*": { + "de": "RDF-Typ" + }, + "type": "string" + }, + "uniqueItems": true + }, + "exact_ontology_match": { + "type": "array", + "title": "Ontology Match (Exact)", + "description": "Only use this property if all properties defined in the Ontology can be asserted to this object.", + "titel*": { + "de": "Ontologieübereinstimmung (exakt)" + }, + "description*": { + "de": "Verwenden Sie diese Eigenschaft nur, wenn alle in der Ontologie definierten Eigenschaften auf dieses Objekt übertragen werden können." + }, + "eval_template": [ + { + "type": "mustache-wikitext", + "mode": "render", + "value": "{{#exact_ontology_match}} {{=<% %>=}} {{Viewer/Link |url= <%={{ }}=%> {{{.}}} {{=<% %>=}} }} <br><%={{ }}=%>{{/exact_ontology_match}}" + } + ], + "items": { + "type": "string", + "options": { + "compact": "true" + } + }, + "uniqueItems": true + }, + "close_ontology_match": { + "type": "array", + "title": "Ontology Match (Close)", + "description": "Use this property if you are not sure if all properties from the ontology can be asserted to this object but the meaning is very close.", + "titel*": { + "de": "Ontologieübereinstimmung (ähnlich)" + }, + "description*": { + "de": "Verwenden Sie diese Eigenschaft, wenn Sie nicht sicher sind, ob alle Eigenschaften aus der Ontologie für dieses Objekt geltend gemacht werden können, die Bedeutung aber sehr nahe liegt." + }, + "eval_template": [ + { + "type": "mustache-wikitext", + "mode": "render", + "value": "{{#close_ontology_match}} {{=<% %>=}} {{Viewer/Link |url= <%={{ }}=%> {{{.}}} {{=<% %>=}} }} <br><%={{ }}=%>{{/close_ontology_match}}" + } + ], + "items": { + "type": "string", + "options": { + "compact": "true" + } + }, + "uniqueItems": true + }, + "uuid": { + "title": "UUID", + "type": "string", + "format": "uuid", + "options": { + "hidden": true + } + }, + "iri": { + "title": "IRI", + "description": "The Internationalized Resource Identifier (IRI) of this entity", + "description*": { + "de": "Der internationalisierte Ressourcenbezeichner (IRI) dieser Entität" + }, + "type": "string", + "format": "url", + "options": { + "hidden": true + } + }, + "name": { + "title": "Technical name", + "title*": { + "de": "Technischer Name" + }, + "description": "Technical / Machine compatible name", + "description*": { + "de": "Technischer / Maschinenkompatibler Name" + }, + "type": "string", + "options": { + "hidden": true + } + }, + "label": { + "type": "array", + "title": "Label(s)", + "title*": { + "de": "Bezeichnung(en)" + }, + "description": "At least one label is required.", + "description*": { + "de": "Mindestens eine Bezeichnung ist erforderlich." + }, + "format": "table", + "eval_template": [ + { + "type": "mustache-wikitext", + "mode": "render", + "$comment": "Displays value according to user language with eng as fallback option. Note: {{=<% %>=}} changes mustache expression from {{..}} to <% %> for mixing with wikitext templates", + "value": "{{=<% %>=}} {{#switch:{{USERLANGUAGECODE}} <%#label%> | {{#ifeq: <%lang%>|en|#default|<%lang%>}} = <%text%> <%/label%> }}" + } + ], + "items": { + "$ref": "/wiki/JsonSchema:Label?action=raw" + }, + "minItems": 1 + }, + "short_name": { + "type": "array", + "title": "Short name(s)", + "title*": { + "de": "Kurzname(n)" + }, + "description": "Abbreviation, Acronym, etc.", + "description*": { + "de": "Abkürzung, Akronym, etc." + }, + "format": "table", + "eval_template": { + "type": "mustache-wikitext", + "mode": "render", + "$comment": "Displays value according to user language with eng as fallback option. Note: {{=<% %>=}} changes mustache expression from {{..}} to <% %> for mixing with wikitext templates", + "value": "{{=<% %>=}} {{#switch:{{USERLANGUAGECODE}} <%#short_name%> | {{#ifeq: <%lang%>|en|#default|<%lang%>}} = <%text%> <%/short_name%> }}" + }, + "items": { + "title": "ShortName", + "title*": { + "en": "Short name", + "de": "Kurzname" + }, + "$ref": "/wiki/JsonSchema:Label?action=raw" + } + }, + "query_label": { + "title": "Query label", + "title*": { + "de": "Abfrage-Bezeichnung" + }, + "type": "string", + "options": { + "hidden": true, + "conditional_visible": { + "modes": [ + "query" + ] + } + } + }, + "description": { + "title": "Description", + "title*": { + "de": "Beschreibung" + }, + "type": "array", + "format": "table", + "eval_template": { + "type": "mustache-wikitext", + "mode": "render", + "$comment": "Displays value according to user language with eng as fallback option. Note: {{=<% %>=}} changes mustache expression from {{..}} to <% %> for mixing with wikitext templates", + "value": "{{=<% %>=}} {{#switch:{{USERLANGUAGECODE}} <%#description%> | {{#ifeq: <%lang%>|en|#default|<%lang%>}} = <%text%> <%/description%> }}" + }, + "items": { + "$ref": "/wiki/JsonSchema:Description?action=raw" + } + }, + "image": { + "title": "Image", + "title*": { + "de": "Bild" + }, + "propertyOrder": 1020, + "type": "string", + "format": "url", + "options": { + "upload": {} + }, + "links": [ + { + "href": "{{#if self}}/w/index.php?title=Special:Redirect/file/{{self}}&width=200&height=200{{/if}}", + "mediaType": "image" + }, + { + "href": "{{#if self}}/w/index.php?title=Special:Redirect/file/{{self}}{{/if}}", + "rel": "{{#if self}}download{{/if}}", + "download": true + } + ] + }, + "classification_categories": { + "type": "array", + "format": "table", + "title": "Classification categories", + "title*": { + "de": "Ordnungskategorien" + }, + "description": "Classification categories are used to categorize instances, e.g., according to their use but not their properties. When querying for instances of a here listed classification category, this instance will be returned. Note: Classification categories define no properties, while 'regular' categories define properties, which an instance assigns values to.", + "description*": { + "de": "Ordnungskategorien werden verwendet, um Instanzen zu kategorisieren, z. B. nach ihrer Verwendung, nicht aber nach ihren Eigenschaften. Bei der Abfrage nach Instanzen einer hier aufgeführten Ordnungskategorie wird diese Instanz zurückgegeben. Hinweis: Ordnungskategorien definieren keine Eigenschaften, während 'normale' Kategorien Eigenschaften definieren, denen eine Instanz Werte zuweist." + }, + "eval_template": [ + { + "type": "mustache-wikitext", + "mode": "render", + "value": "{{=<% %>=}} <%#classification_categories%> {{Viewer/Link |page=<%.%> }} <br><%/classification_categories%>" + } + ], + "items": { + "title": "Classification category", + "title*": { + "de": "Ordnungskategorie" + }, + "type": "string", + "format": "autocomplete", + "options": { + "autocomplete": { + "$comment": "SubCategories of Item OR OwlThing (Category:OSW379d5a1589c74c82bc0de47938264d00)", + "query": "[[SubClassOf::Category:Item]] OR [[SubClassOf.SubClassOf::Category:Item]] OR [[SubClassOf.SubClassOf.SubClassOf::Category:Item]] OR [[SubClassOf.SubClassOf.SubClassOf.SubClassOf::Category:Item]] OR [[SubClassOf.SubClassOf.SubClassOf.SubClassOf.SubClassOf::Category:Item]] OR [[SubClassOf::Category:OSW379d5a1589c74c82bc0de47938264d00]]" + } + } + } + }, + "keywords": { + "type": "array", + "format": "table", + "title": "Keywords / Tags", + "title*": { + "de": "Schlagworte / Tags" + }, + "description": "Designated to the user defined categorization of this element", + "description*": { + "de": "Dient der nutzerdefinierten Kategorisierung des Elements" + }, + "eval_template": [ + { + "type": "mustache-wikitext", + "mode": "render", + "value": "{{=<% %>=}} <%#keywords%> {{Viewer/Link |page=<%.%> }} <br><%/keywords%>" + } + ], + "items": { + "title": "Keyword", + "title*": { + "de": "Schlagwort" + }, + "type": "string", + "format": "autocomplete", + "range": "Category:OSW09f6cdd54bc54de786eafced5f675cbe" + } + }, + "based_on": { + "type": "array", + "title": "Based on", + "title*": { + "de": "Basierend auf" + }, + "description": "Other entities on which this one is based, e.g. when it is created by copying", + "description*": { + "de": "Andere Entitäten auf die diese basiert, z. B. wenn sie durch Kopieren entstanden ist" + }, + "eval_template": [ + { + "type": "mustache-wikitext", + "mode": "render", + "value": "{{=<% %>=}} <%#based_on%> {{Viewer/Link |page=<%.%> }} <br><%/based_on%>" + } + ], + "items": { + "type": "string", + "format": "autocomplete", + "options": { + "autocomplete": { + "$comment": "All categories here since subclassof_range is limited by depth 5", + "query": "[[:Category:%2B]]" + } + } + }, + "options": { + "hidden": "true" + } + }, + "access_policy": { + "title": "AccessAndUsagePolicy", + "title*": { + "en": "Access policy", + "de": "Zugriffsbeschränkungen" + }, + "description": "Defines access control rules (who can read/write this entity).", + "description*": { + "de": "Definiert Zugriffsregeln (wer darf diese Entität lesen/bearbeiten)." + }, + "type": "object", + "defaultProperties": [ + "usergroup_permissions" + ], + "$comment": "See also https://www.w3.org/TR/odrl-vocab/#term-hasPolicy and https://w3c.github.io/odrl/bp/#. For interoperability with DSP/DCAT see https://docs.internationaldataspaces.org/ids-knowledgebase/dataspace-protocol/catalog/catalog.protocol#id-1.2-dcat-and-odrl-profiles.", + "eval_template": { + "$comment": "See https://www.mediawiki.org/wiki/Extension:Semantic_ACL", + "type": "mustache-wikitext", + "mode": "render", + "value": "{{=<% %>=}} <%#access_policy.usergroup_permissions%> {{#if: <%assignee_usergroup%> | [[<%assignee_usergroup%>]] {{#switch: <%action%> |odrl:read= {{#set: |Visible to=whitelist |Visible to group= {{#ask: [[<%assignee_usergroup%>]]|?HasGroupname=|mainlabel=-|link=none|format=plain}} }} (read) |odrl:write= {{#set: |Editable by=whitelist |Editable by group= {{#ask: [[<%assignee_usergroup%>]]|?HasGroupname=|mainlabel=-|link=none|format=plain}} }} (write) |#default=}} <br> |}} <%/access_policy.usergroup_permissions%> <%#access_policy.user_permissions%> {{#if: <%assignee_user%> | [[<%assignee_user%>]] {{#switch: <%action%> |odrl:read= {{#set: |Visible to=whitelist |Visible to user= User:{{#ask: [[<%assignee_user%>]]|?HasUsername=|mainlabel=-|link=none|format=plain}} }} (read) |odrl:write= {{#set: |Editable by=whitelist |Editable by user= User:{{#ask: [[<%assignee_user%>]]|?HasUsername=|mainlabel=-|link=none|format=plain}} }} (write) |#default=}} <br> |}} <%/access_policy.user_permissions%>" + }, + "properties": { + "usergroup_permissions": { + "title": "GroupPermissions", + "title*": { + "en": "Group permissions", + "de": "Gruppen-Erlaubnisse" + }, + "type": "array", + "format": "table", + "items": { + "title": "GroupPermission", + "title*": { + "en": "Permission", + "de": "Erlaubnis" + }, + "type": "object", + "required": [ + "assignee_usergroup", + "action" + ], + "properties": { + "assignee_usergroup": { + "title": "User group", + "title*": { + "de": "Nutzergruppe" + }, + "type": "string", + "minLength": 1, + "format": "autocomplete", + "range": "Category:OSWb24f37ada8d043c194e7036df5d86b39", + "options": { + "autocomplete": { + "$commment": "user should not create new or edit existing groups here", + "create_inline": false + } + } + }, + "action": { + "type": "string", + "title": "Action", + "title*": { + "de": "Aktion" + }, + "enum": [ + "odrl:read", + "odrl:write" + ], + "options": { + "enum_titles": [ + "read", + "edit" + ], + "enum_titles*": [ + "Lesen", + "Bearbeiten" + ] + } + } + } + } + }, + "user_permissions": { + "title": "UserPermissions", + "title*": { + "en": "User permissions", + "de": "Nutzer-Erlaubnisse" + }, + "type": "array", + "format": "table", + "items": { + "title": "Permission", + "title*": { + "en": "Permission", + "de": "Erlaubnis" + }, + "type": "object", + "required": [ + "assignee_user", + "action" + ], + "properties": { + "assignee_user": { + "title": "Specific user", + "title*": { + "de": "Bestimmter Nutzer" + }, + "type": "string", + "minLength": 1, + "format": "autocomplete", + "range": "Category:OSW8b3e627eb76b46d68fce3a12ff220c8e", + "options": { + "autocomplete": { + "$commment": "user should not create new or edit existing accounts here", + "create_inline": false + } + } + }, + "action": { + "type": "string", + "title": "Action", + "title*": { + "de": "Aktion" + }, + "enum": [ + "odrl:read", + "odrl:write" + ], + "options": { + "enum_titles": [ + "read", + "edit" + ], + "enum_titles*": [ + "Lesen", + "Bearbeiten" + ] + } + } + } + } + } + } + }, + "statements": { + "type": "array", + "title": "Statements", + "title*": { + "de": "Aussagen" + }, + "format": "tabs", + "propertyOrder": 1010, + "eval_template": { + "type": "mustache-wikitext", + "mode": "render", + "value": "{{=<% %>=}} <%#statements%>.. {{Viewer/Link |page=<%#predicate%><%predicate%><%/predicate%><%#property%><%property%><%/property%><%#quantity%><%quantity%><%/quantity%>}} <%#object%>{{Viewer/Link |page=<%object%>}}<%/object%><%#value%><%value%><%/value%><br><%/statements%>" + }, + "items": { + "$ref": "/wiki/JsonSchema:Statement?action=raw" + } + }, + "attachments": { + "type": "array", + "title": "File attachments", + "title*": { + "de": "Dateianhänge" + }, + "eval_template": { + "type": "mustache-wikitext", + "mode": "render", + "value": "{{=<% %>=}} <%={{ }}=%> {{#attachments}}{{{.}}};{{/attachments}} {{=<% %>=}} <%={{ }}=%>" + }, + "options": { + "collapsed": true + }, + "propertyOrder": 1030, + "items": { + "title": "Attachment", + "title*": { + "de": "Anhang" + }, + "type": "string", + "format": "url", + "options": { + "upload": {} + }, + "links": [ + { + "href": "{{#if self}}/w/index.php?title=Special:Redirect/file/{{self}}{{/if}}", + "rel": "{{#if self}}download{{/if}}", + "download": true + } + ] + } + }, + "meta": { + "$ref": "/wiki/JsonSchema:Meta?action=raw", + "options": { + "hidden": true + } + } + } +} diff --git a/tests/integration/store_and_load_test.py b/tests/integration/store_and_load_test.py index 2ba833ef..b38ac5cd 100644 --- a/tests/integration/store_and_load_test.py +++ b/tests/integration/store_and_load_test.py @@ -215,6 +215,7 @@ def test_characteristic_creation(wiki_domain, wiki_username, wiki_password): my_characteristic = model.CharacteristicType( uuid="efad8086-4a76-47ca-b278-f5f944e5754b", name="TestCharacteristic", + subclass_of=["Category:OSW93ccae36243542ceac6c951450a81d47"], # Characteristic label=[model.Label(text="Test Characteristic")], description=[ model.Description(text="A test characteristic for the osw python package") diff --git a/tests/test_cli.py b/tests/test_cli.py index c7d8168b..117f093c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1010,3 +1010,46 @@ def test_misspelled_command_option_keeps_clicks_suggestion(runner): combined = _usage_error(result) assert "--jsondata" in combined assert "before the command" not in combined + + +def test_entity_put_reports_the_reason_of_a_failed_store( + runner, configured_env, monkeypatch +): + """The CLI shows why store_entity refused, not just the failed titles.""" + from opensemantic.core.v1 import AnnotationProperty + + from osw.core import OSW + from osw.service.ops import entities + + fake_osw = MagicMock() + fake_osw.fetch_schema.return_value = MagicMock(error_messages=[]) + fake_osw.store_entity.side_effect = OSW.StoreEntityPartialError( + OSW.StoreEntityResult( + change_id="c1", + pages={}, + failed={ + "Property:NewName": ValueError( + "move the page 'Property:OldName' to 'Property:NewName'" + ) + }, + ) + ) + monkeypatch.setattr("osw.service.context.OswExpress", lambda **kwargs: fake_osw) + monkeypatch.setattr( + entities, "_resolve_category_class", lambda category: AnnotationProperty + ) + + result = runner.invoke( + app, + [ + "entity", + "put", + "Category:Property", + "--jsondata", + '{"name": "NewName", "label": [{"text": "x"}]}', + ], + ) + + assert result.exit_code == 1 + assert "move the page" in result.stderr + assert "Property:OldName" in result.stderr diff --git a/tests/test_controller_type_defaults.py b/tests/test_controller_type_defaults.py new file mode 100644 index 00000000..0cd7976c --- /dev/null +++ b/tests/test_controller_type_defaults.py @@ -0,0 +1,34 @@ +"""Pins that every file controller keeps the OSW category of its wrapped model. + +A controller's 'type' field decides which wiki category an entity is stored +under. If a controller does not redefine 'type', it can silently inherit a +different category from earlier in its MRO, filing pages under the wrong +class. +""" + +import osw.model.entity as model +from osw.controller.file.local import LocalFileController +from osw.controller.file.memory import InMemoryController +from osw.controller.file.wiki import WikiFileController + + +def test_local_file_controller_keeps_local_files_category(): + assert ( + LocalFileController.__fields__["type"].get_default() + == model.LocalFile.__fields__["type"].get_default() + ) + + +def test_in_memory_controller_keeps_local_files_category(): + """Same base order as LocalFileController, so the same category applies.""" + assert ( + InMemoryController.__fields__["type"].get_default() + == model.LocalFile.__fields__["type"].get_default() + ) + + +def test_wiki_file_controller_keeps_wiki_files_category(): + assert ( + WikiFileController.__fields__["type"].get_default() + == model.WikiFile.__fields__["type"].get_default() + ) diff --git a/tests/test_express_domain.py b/tests/test_express_domain.py new file mode 100644 index 00000000..d7e0c6a9 --- /dev/null +++ b/tests/test_express_domain.py @@ -0,0 +1,117 @@ +"""OswExpress accepts an http(s) scheme prefix and local hosts with a port.""" + +from unittest.mock import MagicMock + +import pytest + +import osw.express + + +@pytest.fixture +def connect(mocker): + """Build an OswExpress with the network calls replaced by fakes. + + Returns a function that takes the domain and returns the instance, the URL + of the reachability check and the iri passed to WtSite. + """ + + def _connect(domain): + get = mocker.patch( + "osw.express.requests.get", return_value=MagicMock(status_code=200) + ) + wtsite = mocker.patch( + "osw.express.WtSite", + MagicMock(return_value=MagicMock(spec=osw.express.WtSite)), + ) + instance = osw.express.OswExpress( + domain=domain, cred_mngr=MagicMock(cred_filepath=None) + ) + probe_url = get.call_args.args[0] + iri = wtsite.WtSiteConfig.call_args.kwargs["iri"] + return instance, probe_url, iri + + return _connect + + +@pytest.mark.parametrize( + "given,domain,scheme,probe_url,iri", + [ + ( + "wiki.example.org", + "wiki.example.org", + "https", + "https://wiki.example.org/wiki/Main_Page", + "wiki.example.org", + ), + ( + "https://wiki.example.org/", + "wiki.example.org", + "https", + "https://wiki.example.org/wiki/Main_Page", + "wiki.example.org", + ), + ( + "localhost:8081", + "localhost:8081", + "https", + "https://localhost:8081/wiki/Main_Page", + "localhost:8081", + ), + ( + "http://localhost:8081", + "localhost:8081", + "http", + "http://localhost:8081/wiki/Main_Page", + "http://localhost:8081", + ), + ( + "HTTP://127.0.0.1:8081", + "127.0.0.1:8081", + "http", + "http://127.0.0.1:8081/wiki/Main_Page", + "http://127.0.0.1:8081", + ), + ], +) +def test_scheme_prefix_and_local_hosts(connect, given, domain, scheme, probe_url, iri): + instance, probed, site_iri = connect(given) + assert instance.domain == domain + assert instance.scheme == scheme + assert probed == probe_url + assert site_iri == iri + + +@pytest.mark.parametrize( + "domain,valid", + [ + ("wiki-dev.open-semantic-lab.org", True), + ("sub.domain.co.uk", True), + ("localhost", True), + ("localhost:8081", True), + ("127.0.0.1:8081", True), + ("mediawiki", True), + ("http://localhost:8081", False), + ("not a domain!", False), + ("-invalid.com", False), + ("invalid-.com", False), + ("wiki.example.org:port", False), + ("", False), + ], +) +def test_validate_domain(domain, valid): + if valid: + assert osw.express.OswExpress.validate_domain(domain) == domain + else: + with pytest.raises(AssertionError): + osw.express.OswExpress.validate_domain(domain) + + +@pytest.mark.parametrize( + "scheme,valid", [("http", True), ("https", True), ("ftp", False)] +) +def test_validate_scheme(scheme, valid): + if valid: + assert osw.express.OswExpress.validate_scheme(scheme) == scheme + else: + with pytest.raises(AssertionError): + osw.express.OswExpress.validate_scheme(scheme) diff --git a/tests/test_fetch_schema_cache.py b/tests/test_fetch_schema_cache.py new file mode 100644 index 00000000..24972d30 --- /dev/null +++ b/tests/test_fetch_schema_cache.py @@ -0,0 +1,136 @@ +"""Unit tests for the page cache handling of OSW.fetch_schema(). + +Regression guard for #176: the cache state was taken and restored once per +schema title inside _fetch_schema(). With two or more titles the second +snapshot already read the state the first call had set, so the restore never +disabled the cache again. An early return or an exception skipped the restore +as well. fetch_schema() must now take the state once and restore it in every +case. +""" + +import threading + +import pytest + +from osw.core import OSW +from osw.wtsite import WtSite + + +def _make_fake_wtsite(cache_enabled: bool) -> WtSite: + """A WtSite that performs no network calls, with a known cache state.""" + ws = WtSite.__new__(WtSite) + ws._session_lock = threading.RLock() + ws._page_cache = {} + ws._cache_enabled = cache_enabled + return ws + + +def _make_osw(cache_enabled: bool) -> OSW: + """An OSW bound to that WtSite, bypassing __init__ and validation.""" + return OSW.construct(site=_make_fake_wtsite(cache_enabled)) + + +def _stub_fetch_schema( + monkeypatch, seen_states: list, fail_on: str = None, leaks: bool = False +): + """Replace the per-title worker. Records the cache state it is called with. + + With leaks=True the worker enables the cache and never restores it, which is + what the previous _fetch_schema() did for every title but the last one. + """ + + def stub(self, fetchSchemaParam=None): + seen_states.append(self.site.get_cache_enabled()) + if leaks: + self.site.enable_cache() + if fail_on is not None and fetchSchemaParam.schema_title == fail_on: + raise RuntimeError("fetching the schema failed") + return OSW.FetchSchemaResult( + fetched_schema_titles=[fetchSchemaParam.schema_title] + ) + + monkeypatch.setattr(OSW, "_fetch_schema", stub) + + +def test_cache_is_disabled_again_after_several_titles(monkeypatch): + osw_obj = _make_osw(cache_enabled=False) + seen_states = [] + _stub_fetch_schema(monkeypatch, seen_states) + + osw_obj.fetch_schema( + OSW.FetchSchemaParam(schema_title=["Category:Item", "Category:Entity"]) + ) + + assert seen_states == [True, True] + assert osw_obj.site.get_cache_enabled() is False + + +def test_cache_is_disabled_again_after_a_single_title(monkeypatch): + osw_obj = _make_osw(cache_enabled=False) + seen_states = [] + _stub_fetch_schema(monkeypatch, seen_states) + + osw_obj.fetch_schema(OSW.FetchSchemaParam(schema_title="Category:Item")) + + assert seen_states == [True] + assert osw_obj.site.get_cache_enabled() is False + + +def test_cache_stays_enabled_if_the_caller_had_it_enabled(monkeypatch): + osw_obj = _make_osw(cache_enabled=True) + seen_states = [] + _stub_fetch_schema(monkeypatch, seen_states) + + osw_obj.fetch_schema( + OSW.FetchSchemaParam(schema_title=["Category:Item", "Category:Entity"]) + ) + + assert seen_states == [True, True] + assert osw_obj.site.get_cache_enabled() is True + + +def test_a_worker_that_leaves_the_cache_enabled_does_not_leak(monkeypatch): + """The reported defect: the per-title worker enabled the cache and kept it.""" + osw_obj = _make_osw(cache_enabled=False) + seen_states = [] + _stub_fetch_schema(monkeypatch, seen_states, leaks=True) + + osw_obj.fetch_schema( + OSW.FetchSchemaParam(schema_title=["Category:Item", "Category:Entity"]) + ) + + assert osw_obj.site.get_cache_enabled() is False + + +def test_cache_is_restored_when_a_title_raises(monkeypatch): + osw_obj = _make_osw(cache_enabled=False) + seen_states = [] + _stub_fetch_schema(monkeypatch, seen_states, fail_on="Category:Entity", leaks=True) + + with pytest.raises(RuntimeError): + osw_obj.fetch_schema( + OSW.FetchSchemaParam(schema_title=["Category:Item", "Category:Entity"]) + ) + + assert osw_obj.site.get_cache_enabled() is False + + +def test_cache_is_restored_when_the_last_title_returns_early(monkeypatch): + """A missing schema page returns before the end of _fetch_schema().""" + osw_obj = _make_osw(cache_enabled=False) + + def stub(self, fetchSchemaParam=None): + # mirrors the early return for a page that does not exist, which happens + # after the previous _fetch_schema() had enabled the cache + self.site.enable_cache() + return OSW.FetchSchemaResult( + error_messages=[f"Page {fetchSchemaParam.schema_title} does not exist"] + ) + + monkeypatch.setattr(OSW, "_fetch_schema", stub) + + osw_obj.fetch_schema( + OSW.FetchSchemaParam(schema_title=["Category:Item", "Category:Missing"]) + ) + + assert osw_obj.site.get_cache_enabled() is False diff --git a/tests/test_in_memory_upload.py b/tests/test_in_memory_upload.py index 9713c06f..6d438a9f 100644 --- a/tests/test_in_memory_upload.py +++ b/tests/test_in_memory_upload.py @@ -20,7 +20,10 @@ def test_controller_accepts_a_caller_supplied_stream(): stream = BytesIO(b"payload") controller = InMemoryController(stream=stream) - assert controller.get() is stream + assert controller.stream is stream + # get() hands out a reader over the buffer, not the buffer itself, so a + # caller closing what it read does not empty the controller + assert controller.get().read() == b"payload" assert controller.get().read() == b"payload" @@ -68,7 +71,7 @@ def _fake_from_other(other, osw, **data): controller = seen["source_file_controller"] assert isinstance(controller, InMemoryController) - assert controller.get() is stream + assert controller.stream is stream def test_upload_rejects_a_source_that_is_not_file_like(): diff --git a/tests/test_load_entity_bad_category.py b/tests/test_load_entity_bad_category.py new file mode 100644 index 00000000..79caefaf --- /dev/null +++ b/tests/test_load_entity_bad_category.py @@ -0,0 +1,154 @@ +"""Unit tests for load_entity() when a category page has no usable schema. + +Regression guard for #202: https://github.com/OpenSemanticLab/osw-python/issues/202 +load_entity() read each category's schema in a per-page loop without a guard +around it. A category page with no jsonschema slot, a schema without a +"title" key, or a title that is not a string all raised out of the loop and +ended the whole call, so no further title in a multi-title load was ever +processed. + +These run fully offline: the fake site never touches the network. +""" + +import json +from types import SimpleNamespace + +import osw.model.entity as model +from osw.core import OSW +from osw.utils.wiki import remove_empty + +_CATEGORY = "Category:OSWBadCategoryTest0000000000000000000" + + +class _FakePage: + """A page that serves a fixed jsondata or jsonschema slot.""" + + def __init__(self, title, jsondata=None, schema=None): + self.title = title + self._jsondata = jsondata + self._schema = schema + + def get_slot_content(self, slot): + if slot == "jsondata": + return self._jsondata + if slot == "jsonschema": + return self._schema + return None + + +class _FakeSite: + """A site that resolves get_page() by title, so a page fetch and a + category (schema) fetch return different content.""" + + def __init__(self, pages_by_title, cache_enabled=False): + self._pages_by_title = pages_by_title + self.cache_enabled = cache_enabled + + def get_cache_enabled(self): + return self.cache_enabled + + def enable_cache(self): + self.cache_enabled = True + + def disable_cache(self): + self.cache_enabled = False + + def get_page(self, param): + pages = [self._pages_by_title[title] for title in param.titles] + return SimpleNamespace(pages=pages) + + +def _bad_category_site(title, schema): + """A site with one page whose only category serves the given schema.""" + jsondata = {"type": [_CATEGORY], "uuid": "00000000-0000-0000-0000-000000000000"} + entity_page = _FakePage(title, jsondata=jsondata) + category_page = _FakePage(_CATEGORY, schema=schema) + return _FakeSite({title: entity_page, _CATEGORY: category_page}) + + +def test_missing_jsonschema_slot_does_not_raise(): + """get_slot_content("jsonschema") returning None must not raise.""" + title = "Item:BadNoSlot" + site = _bad_category_site(title, schema=None) + osw_obj = OSW.construct(site=site) + + result = osw_obj.load_entity(title) + + assert result is None + + +def test_schema_without_title_key_does_not_raise(): + """A schema dict with no "title" key must not raise.""" + title = "Item:BadNoTitle" + site = _bad_category_site(title, schema={}) + osw_obj = OSW.construct(site=site) + + result = osw_obj.load_entity(title) + + assert result is None + + +def test_schema_title_not_a_string_does_not_raise(): + """A "title" that is not a string must not raise.""" + title = "Item:BadTitleType" + site = _bad_category_site(title, schema={"title": 123}) + osw_obj = OSW.construct(site=site) + + result = osw_obj.load_entity(title) + + assert result is None + + +def _valid_item_page(title): + """A page whose category and jsondata build a real model.Item entity.""" + item = model.Item(label=[model.Label(text="Test Item")]) + category = item.type[0] + jsondata = json.loads(item.json(exclude_none=True)) + remove_empty(jsondata) + entity_page = _FakePage(title, jsondata=jsondata) + category_page = _FakePage(category, schema={"title": "Item"}) + return entity_page, category_page, item + + +def test_bad_category_on_first_page_does_not_block_the_second(): + """A multi-title load must not lose every title because one page's + category is unusable; the rest of the loop must still be processed.""" + bad_title = "Item:BadFirst" + good_title = "Item:GoodSecond" + + bad_jsondata = { + "type": [_CATEGORY], + "uuid": "11111111-1111-1111-1111-111111111111", + } + bad_entity_page = _FakePage(bad_title, jsondata=bad_jsondata) + bad_category_page = _FakePage(_CATEGORY, schema=None) + + good_entity_page, good_category_page, item = _valid_item_page(good_title) + + site = _FakeSite({ + bad_title: bad_entity_page, + good_title: good_entity_page, + _CATEGORY: bad_category_page, + good_category_page.title: good_category_page, + }) + osw_obj = OSW.construct(site=site) + + result = osw_obj.load_entity([bad_title, good_title]) + + assert len(result) == 1 + assert result[0].uuid == item.uuid + + +def test_bad_category_logs_the_page_title_and_the_category(caplog): + """The logged error must name both the failing page and the category, so + the fault can be traced back to the wiki page that needs a fix.""" + title = "Item:BadLogged" + site = _bad_category_site(title, schema=None) + osw_obj = OSW.construct(site=site) + + osw_obj.load_entity(title) + + assert any( + title in record.message and _CATEGORY in record.message + for record in caplog.records + ) diff --git a/tests/test_load_entity_registered_class.py b/tests/test_load_entity_registered_class.py new file mode 100644 index 00000000..5b71849b --- /dev/null +++ b/tests/test_load_entity_registered_class.py @@ -0,0 +1,441 @@ +"""Unit tests for OSW.load_entity() preferring an already registered class. + +Regression guard for #138: load_entity() used to decide whether to compile a +class for a category by checking ``hasattr(model, cls_name)``, keyed by class +name and only looking inside ``osw.model.entity``. This missed classes from +packaged modules (e.g. ``opensemantic.base.v1._model.Database``) that are +already registered for that category IRI in oold's type registry +(``oold.model.v1._types``), causing load_entity() to silently compile and use +a different, incomplete class instead. + +These tests run fully offline: WtSite.get_page() is fed pages through +``offline_pages`` (see ``osw.wtsite.WtSite.GetPageParam``), so no network or +wiki credentials are required. +""" + +import json +import threading +import uuid as uuid_module +from typing import Any, Dict, Union + +from oold.model.v1 import _types as oold_type_registry +from opensemantic.base.v1 import Database + +import osw.model.entity as model +from osw.core import OSW +from osw.utils.wiki import remove_empty +from osw.wtsite import WtPage, WtSite + + +class OfflineWtPage(WtPage): + """A WtPage that pretends to exist without touching a wiki.""" + + def __init__(self, wtSite: Any = None, title: str = None): + self.wtSite = wtSite + self.title = title + self.exists = True + self._original_content = "" + self.changed: bool = False + self._dict = [] + self._slots: Dict[str, Union[str, dict]] = {"main": ""} + self._slots_changed: Dict[str, bool] = {"main": False} + self._content_model: Dict[str, str] = {"main": "wikitext"} + + +class _FakeConnection: + """Just enough of a requests session for WtSite._clear_cookies().""" + + cookies = [] + + +class _FakeMwSite: + connection = _FakeConnection() + + +def make_offline_wtsite() -> WtSite: + """A WtSite that never touches the network (bypasses __init__).""" + ws = WtSite.__new__(WtSite) + ws._page_cache = {} + ws._cache_enabled = False + ws._session_lock = threading.RLock() + ws._site = _FakeMwSite() + return ws + + +def make_page_for_entity(entity) -> OfflineWtPage: + """Build an offline page whose jsondata slot holds the serialized entity.""" + page = OfflineWtPage(title=f"Item:{OSW.get_osw_id(entity.uuid)}") + jsondata = json.loads(entity.json(exclude_none=True)) + remove_empty(jsondata) + page.set_slot_content("jsondata", jsondata) + return page + + +def make_schema_page(category: str, cls_name: str) -> OfflineWtPage: + """Build an offline page holding the jsonschema slot for a category.""" + page = OfflineWtPage(title=category) + page.set_slot_content("jsonschema", {"title": cls_name}) + return page + + +def make_isolated_cls(name: str, base=model.Item): + """Build a model.Item subclass registered under its own private category IRI. + + Overriding schema_extra's title/uuid makes get_cls_iri() derive a fresh + "Category:OSW<uuid>" IRI for this class alone, so defining it cannot + clobber the registration of any real category (e.g. "Category:Item"). + """ + namespace = { + "Config": type( + "Config", + (base.Config,), + { + "schema_extra": { + **base.Config.schema_extra, + "title": name, + "uuid": str(uuid_module.uuid4()), + } + }, + ), + "__qualname__": name, + } + return type(base)(name, (base,), namespace) + + +def make_generated_cls(name: str): + """Like make_isolated_cls, but defined in osw.model.entity, as a class that + fetch_schema() generated is.""" + cls = make_isolated_cls(name) + cls.__module__ = model.__name__ + return cls + + +def test_load_entity_prefers_registered_class_over_generated_one(): + """A class already registered for the category IRI is used as-is, and no + replacement class is compiled into osw.model.entity for it.""" + assert not hasattr(model, "Database") + + db = Database(name="TestDb", label=[model.Label(text="Test Db")]) + category = db.type[0] + entity_page = make_page_for_entity(db) + schema_page = make_schema_page(category, "Database") + + osw_obj = OSW(site=make_offline_wtsite()) + + result = osw_obj.load_entity( + OSW.LoadEntityParam( + titles=[entity_page.title], + autofetch_schema=True, + offline_pages={ + entity_page.title: entity_page, + category: schema_page, + }, + ) + ) + + entity = result.entities[0] + assert type(entity) is Database + # the packaged class was used directly, nothing was compiled + assert not hasattr(model, "Database") + + +def test_load_entity_falls_back_to_generated_class_when_nothing_registered(): + """A category with nothing registered in oold's type registry still gets + the class already present in osw.model.entity, exactly like before.""" + category = "Category:OSWFakeCategoryNotRegistered00000000000000" + cls_name = "FakeGeneratedClass" + assert oold_type_registry.get(category) is None + + fake_cls = make_isolated_cls(cls_name) + setattr(model, cls_name, fake_cls) + try: + entity_page = OfflineWtPage(title="Item:OSWFakeEntity0000000000000000000000000") + jsondata = { + "type": [category], + "uuid": "00000000-0000-0000-0000-000000000000", + "name": "x", + "label": [{"text": "x"}], + } + remove_empty(jsondata) + entity_page.set_slot_content("jsondata", jsondata) + schema_page = make_schema_page(category, cls_name) + + osw_obj = OSW(site=make_offline_wtsite()) + + result = osw_obj.load_entity( + OSW.LoadEntityParam( + titles=[entity_page.title], + autofetch_schema=True, + offline_pages={ + entity_page.title: entity_page, + category: schema_page, + }, + ) + ) + + entity = result.entities[0] + assert type(entity) is fake_cls + finally: + delattr(model, cls_name) + + +def test_load_entity_ignores_a_registered_subclass_of_the_model_class(caplog): + """osw's own controllers and result wrappers (WikiFileController, + UploadFileResult, ...) inherit the category IRI of the model class they + extend, and oold's registry keeps whichever class was defined last. Such a + specialization needs fields a plain page does not carry, so load_entity() + must fall back to the canonical class in osw.model.entity. + + This is the expected case on every WikiFile load, so it must not raise the + registry conflict warning: nothing claims the slot, it still holds the + specialization. + """ + category = "Category:OSWSubclassTest000000000000000000000000" + cls_name = "SubclassTestBase" + + base_cls = make_isolated_cls(cls_name) + # a controller-like specialization that additionally requires a field the + # page's jsondata does not provide, mirroring UploadFileResult.source + specialized_cls = type(base_cls)( + "SubclassTestController", + (base_cls,), + {"__annotations__": {"source": str}, "__qualname__": "SubclassTestController"}, + ) + setattr(model, cls_name, base_cls) + oold_type_registry[category] = specialized_cls + try: + entity_page = OfflineWtPage( + title="Item:OSWSubclassEntity0000000000000000000000000" + ) + jsondata = { + "type": [category], + "uuid": "22222222-2222-2222-2222-222222222222", + "name": "x", + "label": [{"text": "x"}], + } + remove_empty(jsondata) + entity_page.set_slot_content("jsondata", jsondata) + schema_page = make_schema_page(category, cls_name) + + osw_obj = OSW(site=make_offline_wtsite()) + + result = osw_obj.load_entity( + OSW.LoadEntityParam( + titles=[entity_page.title], + autofetch_schema=True, + offline_pages={ + entity_page.title: entity_page, + category: schema_page, + }, + ) + ) + + assert type(result.entities[0]) is base_cls + assert not any( + "claims the oold type registry slot" in record.message + for record in caplog.records + ) + finally: + delattr(model, cls_name) + oold_type_registry.pop(category, None) + + +def test_load_entity_warns_on_registry_conflict(monkeypatch, caplog): + """If the class about to be used for a category differs from whatever is + now registered for that IRI, load_entity() logs a warning instead of + silently letting the mismatch pass.""" + category = "Category:OSWConflictTest000000000000000000000000" + cls_name = "ConflictGeneratedClass" + assert oold_type_registry.get(category) is None + + generated_cls = make_isolated_cls(cls_name) + other_cls = make_isolated_cls("OtherRegisteredClass") + + def fake_fetch_schema(self, fetchSchemaParam=None): + # Simulate fetch_schema() compiling a class and importing it into + # osw.model.entity, while a *different* class ends up holding the + # oold registry slot for the same category. + setattr(model, cls_name, generated_cls) + oold_type_registry[category] = other_cls + + monkeypatch.setattr(OSW, "fetch_schema", fake_fetch_schema) + + try: + entity_page = OfflineWtPage( + title="Item:OSWConflictEntity00000000000000000000000000" + ) + jsondata = { + "type": [category], + "uuid": "11111111-1111-1111-1111-111111111111", + "name": "x", + "label": [{"text": "x"}], + } + remove_empty(jsondata) + entity_page.set_slot_content("jsondata", jsondata) + schema_page = make_schema_page(category, cls_name) + + osw_obj = OSW(site=make_offline_wtsite()) + + result = osw_obj.load_entity( + OSW.LoadEntityParam( + titles=[entity_page.title], + autofetch_schema=True, + offline_pages={ + entity_page.title: entity_page, + category: schema_page, + }, + ) + ) + + entity = result.entities[0] + assert type(entity) is generated_cls + assert any( + "claims the oold type registry slot" in record.message + for record in caplog.records + ) + finally: + if hasattr(model, cls_name): + delattr(model, cls_name) + oold_type_registry.pop(category, None) + + +def test_load_entity_uses_the_current_class_after_a_reload(monkeypatch): + """fetch_schema() reloads osw.model.entity, which replaces every class + defined there. On a page with two categories, a fetch for the second one + must not leave the entity built from the first one's replaced class.""" + cat_a = "Category:OSWReloadTestA000000000000000000000000" + cat_b = "Category:OSWReloadTestB000000000000000000000000" + name_a, name_b = "ReloadTestA", "ReloadTestB" + + setattr(model, name_a, make_generated_cls(name_a)) + + def fake_fetch_schema(self, fetchSchemaParam=None): + # what importlib.reload(model) does: the requested class appears, and + # every class already generated there becomes a new object + setattr(model, name_a, make_generated_cls(name_a)) + setattr(model, name_b, make_generated_cls(name_b)) + + monkeypatch.setattr(OSW, "fetch_schema", fake_fetch_schema) + try: + entity_page = OfflineWtPage( + title="Item:OSWReloadEntity00000000000000000000000000000" + ) + jsondata = { + "type": [cat_a, cat_b], + "uuid": "33333333-3333-3333-3333-333333333333", + "name": "x", + "label": [{"text": "x"}], + } + remove_empty(jsondata) + entity_page.set_slot_content("jsondata", jsondata) + + osw_obj = OSW(site=make_offline_wtsite()) + + result = osw_obj.load_entity( + OSW.LoadEntityParam( + titles=[entity_page.title], + autofetch_schema=True, + offline_pages={ + entity_page.title: entity_page, + cat_a: make_schema_page(cat_a, name_a), + cat_b: make_schema_page(cat_b, name_b), + }, + ) + ) + + entity = result.entities[0] + assert isinstance(entity, getattr(model, name_a)) + assert isinstance(entity, getattr(model, name_b)) + finally: + for name in (name_a, name_b): + if hasattr(model, name): + delattr(model, name) + oold_type_registry.pop(cat_a, None) + oold_type_registry.pop(cat_b, None) + + +def _load_single_category_page(category: str, cls_name: str, suffix: str): + """Load a page of one category whose schema title is cls_name.""" + entity_page = OfflineWtPage(title=f"Item:OSWNonClass{suffix}".ljust(44, "0")) + jsondata = { + "type": [category], + "uuid": "44444444-4444-4444-4444-444444444444", + "name": "x", + "label": [{"text": "x"}], + } + remove_empty(jsondata) + entity_page.set_slot_content("jsondata", jsondata) + return OSW(site=make_offline_wtsite()).load_entity( + OSW.LoadEntityParam( + titles=[entity_page.title], + autofetch_schema=True, + offline_pages={ + entity_page.title: entity_page, + category: make_schema_page(category, cls_name), + }, + ) + ) + + +def test_load_entity_uses_the_registered_class_when_the_name_is_not_a_class(): + """A schema title can collide with a module attribute of osw.model.entity + that is not a class. The subclass guard must not call issubclass() on it, + and the registered class is used as usual.""" + category = "Category:OSWNonClassRegistered00000000000000000" + cls_name = "NonClassAttributeA" + registered_cls = make_isolated_cls("NonClassRegistered") + setattr(model, cls_name, "not a class") + oold_type_registry[category] = registered_cls + try: + result = _load_single_category_page(category, cls_name, "A") + assert type(result.entities[0]) is registered_cls + finally: + delattr(model, cls_name) + oold_type_registry.pop(category, None) + + +def test_load_entity_logs_a_schema_title_that_names_no_class(caplog): + """With nothing registered, a schema title that names a non-class attribute + of osw.model.entity cannot build an entity. load_entity() logs the failed + construction and returns, as on main, instead of raising.""" + category = "Category:OSWNonClassNotRegistered0000000000000000" + cls_name = "NonClassAttributeB" + assert oold_type_registry.get(category) is None + setattr(model, cls_name, "not a class") + try: + _load_single_category_page(category, cls_name, "B") + assert any( + "Error creating entity from page Item:OSWNonClassB" in record.message + for record in caplog.records + ) + finally: + delattr(model, cls_name) + + +def test_load_entity_logs_a_fetch_that_leaves_no_class_under_the_title( + monkeypatch, caplog +): + """fetch_schema() can register a class for the category while the schema + title still names a non-class attribute of osw.model.entity. The registry + conflict check must not call issubclass() on that value; the failed + construction is logged, as on main.""" + category = "Category:OSWNonClassFetched000000000000000000000" + cls_name = "NonClassAttributeC" + assert oold_type_registry.get(category) is None + fetched_cls = make_isolated_cls("NonClassFetched") + + def fake_fetch_schema(self, fetchSchemaParam=None): + setattr(model, cls_name, "not a class") + oold_type_registry[category] = fetched_cls + + monkeypatch.setattr(OSW, "fetch_schema", fake_fetch_schema) + try: + _load_single_category_page(category, cls_name, "C") + assert any( + "Error creating entity from page Item:OSWNonClassC" in record.message + for record in caplog.records + ) + finally: + if hasattr(model, cls_name): + delattr(model, cls_name) + oold_type_registry.pop(category, None) diff --git a/tests/test_overwrite_policy.py b/tests/test_overwrite_policy.py index 3508888e..5f514a9a 100644 --- a/tests/test_overwrite_policy.py +++ b/tests/test_overwrite_policy.py @@ -19,7 +19,12 @@ import pytest import osw.model.entity as model -from osw.core import OSW, AddOverwriteClassOptions, OverwriteOptions +from osw.core import ( + OSW, + AddOverwriteClassOptions, + OverwriteOptions, + _explicitly_set_empty, +) from osw.utils.wiki import remove_empty from osw.wtsite import WtPage, WtSite @@ -155,6 +160,111 @@ def test_remote_property_unknown_to_the_model_is_preserved(remote_and_local): assert result["name"] == "LocalName" +# -------------------------------------------------------------------------- +# clearing a property +# -------------------------------------------------------------------------- + + +@pytest.fixture +def remote_item_with_description(): + return model.Item( + label=[model.Label(text="Remote label")], + name="RemoteName", + description=[model.Label(text="Remote description")], + ) + + +def test_an_explicitly_emptied_property_clears_the_remote_value( + remote_item_with_description, +): + remote = remote_item_with_description + local = model.Item( + uuid=remote.uuid, + type=remote.type, + label=[model.Label(text="Local label")], + description=[], + ) + policy = OSW.OverwriteClassParam(model=model.Item, overwrite=OverwriteOptions.true) + result = apply_policy(make_remote_page(remote), local, policy) + + assert result["description"] == [] + + +def test_a_property_that_was_never_set_does_not_clear_the_remote_value( + remote_item_with_description, +): + """A field left at its None default never reaches the merge at all.""" + remote = remote_item_with_description + local = model.Item( + uuid=remote.uuid, type=remote.type, label=[model.Label(text="Local label")] + ) + policy = OSW.OverwriteClassParam(model=model.Item, overwrite=OverwriteOptions.true) + result = apply_policy(make_remote_page(remote), local, policy) + + assert result["description"][0]["text"] == "Remote description" + + +def test_an_empty_value_that_was_not_explicitly_set_is_still_stripped( + remote_item_with_description, +): + """remove_empty() still runs: only __fields_set__ decides what survives it. + + The generated models default every field to None, and None is dropped before + remove_empty() is reached, so __fields_set__ has to be edited directly to + produce the one case that tells the two apart: a value that is empty in the + serialized output but was not assigned by the caller. + """ + remote = remote_item_with_description + local = model.Item( + uuid=remote.uuid, + type=remote.type, + label=[model.Label(text="Local label")], + description=[], + ) + local.__fields_set__.discard("description") # as if it had never been assigned + assert json.loads(local.json(exclude_none=True))["description"] == [] + + policy = OSW.OverwriteClassParam(model=model.Item, overwrite=OverwriteOptions.true) + result = apply_policy(make_remote_page(remote), local, policy) + + assert result["description"][0]["text"] == "Remote description" + + +def test_an_explicitly_emptied_property_is_still_protected_by_overwrite_false( + remote_item_with_description, +): + remote = remote_item_with_description + local = model.Item( + uuid=remote.uuid, + type=remote.type, + label=[model.Label(text="Local label")], + description=[], + ) + policy = OSW.OverwriteClassParam(model=model.Item, overwrite=OverwriteOptions.false) + result = apply_policy(make_remote_page(remote), local, policy) + + assert result["description"][0]["text"] == "Remote description" + + +def test_explicitly_set_empty_returns_only_the_fields_set_to_empty(): + entity_with_empty_description = model.Item( + label=[model.Label(text="Local label")], description=[] + ) + jsondata = json.loads(entity_with_empty_description.json(exclude_none=True)) + assert _explicitly_set_empty(entity_with_empty_description, jsondata) == { + "description": [] + } + + entity_without_description = model.Item(label=[model.Label(text="Local label")]) + jsondata_without_description = json.loads( + entity_without_description.json(exclude_none=True) + ) + assert ( + _explicitly_set_empty(entity_without_description, jsondata_without_description) + == {} + ) + + # -------------------------------------------------------------------------- # mutating a policy object after construction # -------------------------------------------------------------------------- @@ -211,6 +321,28 @@ def test_a_reassigned_policy_is_honoured_by_the_merge(remote_and_local): assert result["label"][0]["text"] == "Remote label" +def test_reassigning_model_to_one_missing_a_per_property_key_is_rejected(): + policy = OSW.OverwriteClassParam( + model=model.Item, + overwrite=False, + per_property={"description": OverwriteOptions.true}, + ) + with pytest.raises(ValueError, match="Property not found in model"): + policy.model = model.Label + + assert policy.model is model.Item + assert policy.per_property == {"description": OverwriteOptions.true} + + +def test_reassigning_model_to_one_that_has_the_key_still_applies_the_setting(): + policy = OSW.OverwriteClassParam( + model=model.Item, per_property={"label": OverwriteOptions.true} + ) + policy.model = model.Entity + + assert policy.get_overwrite_setting("label") == OverwriteOptions.true + + # -------------------------------------------------------------------------- # validation of the policy options # -------------------------------------------------------------------------- @@ -426,6 +558,46 @@ def test_two_policies_for_the_same_class_are_rejected(): ) +class _SubItem(model.Item): + """A subclass that does not redefine 'type' and so shares model.Item's.""" + + +def test_a_subclass_sharing_the_parents_type_is_rejected_with_an_explanation(): + with pytest.raises(ValueError, match="subclass that does not redefine"): + OSW.StoreEntityParam( + entities=[model.Item(label=[model.Label(text="x")])], + overwrite_per_class=[ + OSW.OverwriteClassParam(model=model.Item, overwrite=True), + OSW.OverwriteClassParam(model=_SubItem, overwrite=False), + ], + ) + + +def test_the_same_class_declared_twice_is_not_blamed_on_inheritance(): + """issubclass(X, X) is True, so the common typo must not cite a subclass.""" + with pytest.raises(ValueError) as exc_info: + OSW.StoreEntityParam( + entities=[model.Item(label=[model.Label(text="x")])], + overwrite_per_class=[ + OSW.OverwriteClassParam(model=model.Item, overwrite=True), + OSW.OverwriteClassParam(model=model.Item, overwrite=False), + ], + ) + + assert "subclass that does not redefine" not in str(exc_info.value) + + +def test_overwrite_per_class_dispatch_map_is_flat_and_keyed_by_osw_type(): + policy = OSW.OverwriteClassParam(model=model.Item, overwrite=OverwriteOptions.true) + param = OSW.StoreEntityParam( + entities=[model.Item(label=[model.Label(text="x")])], + overwrite_per_class=[policy], + ) + + assert set(param._overwrite_per_class.keys()) == {"Category:Item"} + assert param._overwrite_per_class["Category:Item"].model is model.Item + + # -------------------------------------------------------------------------- # keep existing # -------------------------------------------------------------------------- @@ -519,3 +691,121 @@ def test_keep_existing_does_not_regenerate_the_schema_of_a_category(monkeypatch) page = _store_category(monkeypatch, AddOverwriteClassOptions.keep_existing) assert page.get_slot_content("jsonschema") is None + + +# -------------------------------------------------------------------------- +# skipped pages +# -------------------------------------------------------------------------- + + +def test_keep_existing_reports_a_skipped_page_that_already_existed(monkeypatch): + _stub_page_io(monkeypatch, exists=True) + osw_obj = OSW.construct(site=object()) + + item = model.Item(label=[model.Label(text="Keep me")]) + result = osw_obj.store_entity( + OSW.StoreEntityParam( + entities=[item], + overwrite=AddOverwriteClassOptions.keep_existing, + parallel=False, + ) + ) + + title = f"Item:{OSW.get_osw_id(item.uuid)}" + assert title in result.skipped + assert title in result.pages + + +def test_skipped_pages_are_reported_from_the_parallel_path(monkeypatch): + """store_entity_() fills the dict from worker threads when parallel is set.""" + _stub_page_io(monkeypatch, exists=True) + osw_obj = OSW.construct(site=object()) + + items = [model.Item(label=[model.Label(text=f"Keep {i}")]) for i in range(3)] + result = osw_obj.store_entity( + OSW.StoreEntityParam( + entities=items, + overwrite=AddOverwriteClassOptions.keep_existing, + parallel=True, + ) + ) + + titles = {f"Item:{OSW.get_osw_id(item.uuid)}" for item in items} + assert set(result.skipped) == titles + + +def test_a_newly_created_page_is_not_reported_as_skipped(monkeypatch): + _stub_page_io(monkeypatch, exists=False) + osw_obj = OSW.construct(site=object()) + + item = model.Item(label=[model.Label(text="Create me")]) + result = osw_obj.store_entity( + OSW.StoreEntityParam( + entities=[item], + overwrite=AddOverwriteClassOptions.keep_existing, + parallel=False, + ) + ) + + assert result.skipped == {} + assert f"Item:{OSW.get_osw_id(item.uuid)}" in result.pages + + +def test_an_overwritten_existing_page_is_not_reported_as_skipped(monkeypatch): + _stub_page_io(monkeypatch, exists=True) + osw_obj = OSW.construct(site=object()) + + item = model.Item(label=[model.Label(text="Overwrite me")]) + result = osw_obj.store_entity( + OSW.StoreEntityParam( + entities=[item], overwrite=OverwriteOptions.true, parallel=False + ) + ) + + assert result.skipped == {} + + +def test_a_skipped_page_is_not_verified_and_survives_a_failed_neighbour(monkeypatch): + """A kept page was never edited, so the write verification must ignore it. + + 'keep existing' and the write verification of #175 were written separately + and first meet here. A kept page must not be queried, and must stay in + 'pages' and 'skipped' when the verification reports the other page of the + same call as missing. + """ + existing = model.Item(label=[model.Label(text="Keep me")]) + created = model.Item(label=[model.Label(text="Create me")]) + kept_title = f"Item:{OSW.get_osw_id(existing.uuid)}" + created_title = f"Item:{OSW.get_osw_id(created.uuid)}" + verified = [] + + monkeypatch.setattr( + WtPage, "init", lambda self: setattr(self, "exists", self.title == kept_title) + ) + monkeypatch.setattr( + WtPage, "get_url", lambda self: f"https://example.org/{self.title}" + ) + monkeypatch.setattr(WtPage, "edit", lambda self, *a, **k: None) + + def _missing(self, titles): + verified.extend(titles) + return [created_title] + + monkeypatch.setattr(OSW, "_get_missing_page_titles", _missing) + osw_obj = OSW.construct(site=object()) + + with pytest.raises(OSW.StoreEntityPartialError) as exc_info: + osw_obj.store_entity( + OSW.StoreEntityParam( + entities=[existing, created], + overwrite=AddOverwriteClassOptions.keep_existing, + parallel=False, + ) + ) + + err = exc_info.value + assert verified == [created_title] + assert kept_title in err.result.skipped + assert kept_title in err.result.pages + assert created_title in err.failed + assert err.stored == [] diff --git a/tests/test_property_title.py b/tests/test_property_title.py new file mode 100644 index 00000000..6a607367 --- /dev/null +++ b/tests/test_property_title.py @@ -0,0 +1,179 @@ +"""Unit tests for the page title of Property entities. + +A Property entity is stored as 'Property:<name>' (get_title from the opensemantic +package). Writing a property whose name changed would create a second property +page with the same uuid, so store_entity_() refuses it when the wiki holds the +uuid under a different 'Property:' title. + +These run fully offline: WtPage.init, the overwrite policy, WtPage.edit and the +write verification are stubbed and the site is a MagicMock. +""" + +from unittest.mock import MagicMock +from uuid import uuid4 + +import pytest +from opensemantic.core.v1 import AnnotationProperty, Label + +import osw.model.entity as model +from osw.core import OSW +from osw.service import errors +from osw.service.config import Settings +from osw.service.context import Context, Policy +from osw.service.ops import entities +from osw.wtsite import WtPage + + +@pytest.fixture +def edits(monkeypatch): + """The titles of the pages an edit was sent for.""" + sent = [] + monkeypatch.setattr(WtPage, "init", lambda self: setattr(self, "exists", False)) + monkeypatch.setattr( + OSW, "_apply_overwrite_policy", staticmethod(lambda param: param.page) + ) + monkeypatch.setattr(WtPage, "edit", lambda self, *a, **kw: sent.append(self.title)) + monkeypatch.setattr(OSW, "_get_missing_page_titles", lambda self, titles: []) + return sent + + +def _osw(hits=()): + site = MagicMock() + site.semantic_search.return_value = list(hits) + return OSW.construct(site=site) + + +def _prop(name, **kwargs): + return AnnotationProperty(name=name, label=[Label(text=name)], **kwargs) + + +def _store(osw, entity): + return osw.store_entity(OSW.StoreEntityParam(entities=[entity], parallel=False)) + + +def test_property_without_meta_is_written_to_its_name(edits): + osw = _osw() + _store(osw, _prop("HasName")) + assert edits == ["Property:HasName"] + + +def test_update_with_same_name_passes(edits): + prop = _prop("HasName") + osw = _osw(hits=["Property:HasName"]) + _store(osw, prop) + assert edits == ["Property:HasName"] + osw.site.semantic_search.assert_called_once_with(f"[[HasUuid::{prop.uuid}]]") + + +def test_no_hit_passes(edits): + osw = _osw(hits=[]) + _store(osw, _prop("HasName")) + assert edits == ["Property:HasName"] + + +def test_renamed_property_is_refused(edits): + prop = _prop("NewName") + osw = _osw(hits=["Property:OldName"]) + with pytest.raises(OSW.StoreEntityPartialError) as exc_info: + _store(osw, prop) + (exc,) = exc_info.value.failed.values() + assert isinstance(exc, ValueError) + text = str(exc) + assert "Property:OldName" in text + assert "Property:NewName" in text + assert str(prop.uuid) in text + assert "move" in text + assert edits == [] + + +def test_hit_in_another_namespace_is_ignored(edits): + osw = _osw(hits=["Item:OSW123"]) + _store(osw, _prop("HasName")) + assert edits == ["Property:HasName"] + + +@pytest.mark.parametrize( + "name, hit", + [("hasName", "Property:HasName"), ("Has_Name", "Property:Has Name")], +) +def test_title_comparison_is_normalized_like_mediawiki(edits, name, hit): + osw = _osw(hits=[hit]) + _store(osw, _prop(name)) + assert len(edits) == 1 + + +def test_explicit_meta_title_wins_over_name(edits): + prop = _prop("NewName") + prop.meta = _meta(title="Pinned") + osw = _osw(hits=["Property:Pinned"]) + _store(osw, prop) + assert edits == ["Property:Pinned"] + + +def test_item_keeps_the_uuid_title_and_runs_no_query(edits): + item = model.Item(label=[model.Label(text="Solo")]) + osw = _osw() + _store(osw, item) + assert edits == [f"Item:OSW{str(item.uuid).replace('-', '')}"] + osw.site.semantic_search.assert_not_called() + + +def _meta(title=None, namespace=None): + return model.Meta(wiki_page=model.WikiPage(title=title, namespace=namespace)) + + +# -- delete_entity ----------------------------------------------------------- +def _delete(entity): + site = MagicMock() + page = MagicMock(exists=True) + site.get_page.return_value.pages = [page] + OSW.construct(site=site).delete_entity( + OSW.DeleteEntityParam(entities=[entity], parallel=False) + ) + (call,) = site.get_page.call_args_list + page.delete.assert_called_once() + return call.args[0].titles + + +def test_delete_property_without_meta_targets_its_name(): + assert _delete(_prop("HasName")) == ["Property:HasName"] + + +def test_delete_property_with_meta_title_targets_the_meta_title(): + prop = _prop("HasName") + prop.meta = _meta(title="Pinned") + assert _delete(prop) == ["Property:Pinned"] + + +# -- service op -------------------------------------------------------------- +def test_create_or_update_entity_surfaces_the_rename_refusal(monkeypatch, edits): + osw = _osw(hits=["Property:OldName"]) + monkeypatch.setattr( + OSW, "fetch_schema", lambda self, param: MagicMock(error_messages=[]) + ) + monkeypatch.setattr( + entities, "_resolve_category_class", lambda category: AnnotationProperty + ) + monkeypatch.setattr(entities.config, "get_active_domain", lambda: "w.example.org") + ctx = Context( + Settings(domain="w.example.org", username="u", password="p"), + Policy(), + osw=osw, + ) + with pytest.raises(errors.OpError) as exc_info: + entities.create_or_update_entity( + ctx, + category="Category:OSW0000000000000000000000000000AnnotationProperty", + jsondata={ + "uuid": str(uuid4()), + "name": "NewName", + "label": [{"text": "x"}], + }, + ) + text = str(exc_info.value) + assert "Property:OldName" in text + assert "Property:NewName" in text + assert "move" in text + assert edits == [] + # the MCP wire type stays the one a failed store had before + assert exc_info.value.payload()["type"] == "StoreEntityPartialError" diff --git a/tests/test_service_context.py b/tests/test_service_context.py index 63f89a2a..49cc06e7 100644 --- a/tests/test_service_context.py +++ b/tests/test_service_context.py @@ -111,6 +111,26 @@ def test_osw_property_raises_not_configured_when_no_active_domain( assert "--instance" in str(exc_info.value) +@pytest.mark.parametrize( + "active_iri,connect_domain", + [ + ("http://localhost:8081", "http://localhost:8081"), + ("https://wiki.example.org/w/", "wiki.example.org"), + ("wiki.example.org", "wiki.example.org"), + ], +) +def test_osw_property_keeps_an_http_scheme(monkeypatch, active_iri, connect_domain): + """An http active iri reaches OswExpress with its scheme, https stays bare.""" + monkeypatch.setattr(config, "get_active_iri", lambda: active_iri) + ctx = Context(_settings()) + osw_for = MagicMock() + monkeypatch.setattr(ctx, "osw_for", osw_for) + + _ = ctx.osw + + osw_for.assert_called_once_with(connect_domain) + + # -- limit ---------------------------------------------------------------- def test_limit_falls_back_to_settings_max_results(): ctx = Context(_settings(), osw=object()) diff --git a/tests/test_store_entity_category_templates.py b/tests/test_store_entity_category_templates.py new file mode 100644 index 00000000..a16ba6a2 --- /dev/null +++ b/tests/test_store_entity_category_templates.py @@ -0,0 +1,253 @@ +"""store_entity() must select category templates like the wiki form editor. + +The fixtures are two real Category pages (PhysicalItem and PhysicalItemType) in +the slot state before and after an unchanged save through the form editor of the +dev wiki on 2026-11-05 (revisions 134604 and 134605). The form editor only added +"version": "0.1.0" to each jsondata. + +The test stores the after-jsondata offline on top of the before-jsonschema and +expects the jsonschema the form editor wrote. The meta category pages come from +the package files in tests/fixtures/category_template_selection/meta, so no wiki +is read or written. +""" + +import copy +import json +from pathlib import Path + +import pytest +from opensemantic.core.v1 import Category + +from osw.core import ( + OSW, + collect_category_template_titles, + reset_generated_jsonld_context, + strip_root_members_superseded_by_generated, +) +from osw.wtsite import WtPage + +FIXTURES = Path(__file__).parent / "fixtures" / "category_template_selection" +CHARACTERISTIC_TYPE = "Category:OSWffe74f291d354037b318c422591c5023" + +# fixture file stem -> page title +META_PAGES = { + "Category:Category": "Category", + CHARACTERISTIC_TYPE: "CharacteristicType", + "Category:Entity": "Entity", +} + +PAGES = { + "PhysicalItem": "Category:OSW37a57741ae2e4dd4b29b1172b6848be8", + "PhysicalItemType": "Category:OSWca99a9bb9ad24386b222d18a73c2af5e", +} + + +def _load(name): + return json.loads((FIXTURES / name).read_text(encoding="utf-8")) + + +class _FakeMetaCategoryPage: + def __init__(self, title, stem): + self.title = title + meta = FIXTURES / "meta" + template_file = meta / f"{stem}.schema_template.txt" + self._slots = { + "jsondata": json.loads( + (meta / f"{stem}.jsondata.json").read_text(encoding="utf-8") + ), + "jsonschema": json.loads( + (meta / f"{stem}.jsonschema.json").read_text(encoding="utf-8") + ), + "schema_template": ( + template_file.read_text(encoding="utf-8") + if template_file.exists() + else None + ), + } + + def get_slot_content(self, slot_key): + return self._slots.get(slot_key) + + +class _FakeGetPageResult: + def __init__(self, pages): + self.pages = pages + + +class _FakeSite: + """Serves the meta category pages from the fixture files.""" + + def __init__(self): + self.requested = [] + + def get_page(self, param): + self.requested.extend(param.titles) + return _FakeGetPageResult([ + _FakeMetaCategoryPage(t, META_PAGES[t]) for t in param.titles + ]) + + +def _store_offline(monkeypatch, jsondata, stored_schema): + original_policy = OSW._apply_overwrite_policy + + def policy_with_stored_schema(param): + param.page.set_slot_content("jsonschema", copy.deepcopy(stored_schema)) + return original_policy(param) + + monkeypatch.setattr( + OSW, "_apply_overwrite_policy", staticmethod(policy_with_stored_schema) + ) + site = _FakeSite() + entity = Category(**copy.deepcopy(jsondata)) + result = OSW.construct(site=site).store_entity( + OSW.StoreEntityParam(entities=[entity], namespace="Category", offline=True) + ) + (page,) = result.pages.values() + assert isinstance(page, WtPage) + return page.get_slot_content("jsonschema"), site + + +@pytest.mark.parametrize("name", sorted(PAGES)) +def test_store_matches_form_editor_save(monkeypatch, name): + title = PAGES[name] + before = _load("before-form-editor-save.json")[title] + after = _load("after-form-editor-save.json")[title] + + schema, site = _store_offline(monkeypatch, after["jsondata"], before["jsonschema"]) + + assert schema == after["jsonschema"] + assert CHARACTERISTIC_TYPE in site.requested + + +@pytest.mark.parametrize("name", sorted(PAGES)) +def test_generated_section_lists_both_templates(monkeypatch, name): + title = PAGES[name] + before = _load("before-form-editor-save.json")[title] + after = _load("after-form-editor-save.json")[title] + + schema, _ = _store_offline(monkeypatch, after["jsondata"], before["jsonschema"]) + + comment = schema["$defs"]["generated"]["$comment"] + assert comment.endswith(f"Category:Category {CHARACTERISTIC_TYPE}") + + +def test_collect_follows_allof_in_wiki_order(): + schemas = { + "Category:Category": { + "uuid": "89aafe6d-ae5a-4f29-97ff-df7736d4cab6", + "allOf": [ + {"$ref": f"/wiki/{CHARACTERISTIC_TYPE}?action=raw&slot=jsonschema"} + ], + }, + CHARACTERISTIC_TYPE: { + "uuid": "ffe74f29-1d35-4037-b318-c422591c5023", + "allOf": [{"$ref": "/wiki/Category:Entity?action=raw&slot=jsonschema"}], + }, + "Category:Entity": {"uuid": "ce353767-c628-45bd-9d88-d6eb3009aec0"}, + } + + class Page: + def __init__(self, title): + self.title = title + + def get_slot_content(self, slot): + return schemas[self.title] + + titles = collect_category_template_titles( + lambda ts: [Page(t) for t in ts], ["Category:Category"] + ) + + assert titles == ["Category:Category", CHARACTERISTIC_TYPE] + + +def test_reset_context_keeps_manual_block_when_never_generated(): + schema = {"@context": ["/wiki/Category:A", {"manual": "x"}]} + + reset_generated_jsonld_context(schema, had_generated_section=False) + + assert schema["@context"] == ["/wiki/Category:A", {}, {"manual": "x"}] + + +def test_reset_context_empties_generated_block(): + schema = {"@context": ["/wiki/Category:A", {"gen": "x"}, {"manual": "y"}]} + + reset_generated_jsonld_context(schema, had_generated_section=True) + + assert schema["@context"] == ["/wiki/Category:A", {}, {"manual": "y"}] + + +def test_strip_keeps_manual_siblings(): + root = { + "uuid": "u", + "title": "t", + "properties": {"type": {"default": ["x"], "minItems": 1}}, + } + generated = {"uuid": "u", "properties": {"type": {"default": ["x"]}}} + + assert strip_root_members_superseded_by_generated(root, generated) is True + assert root == {"title": "t", "properties": {"type": {"minItems": 1}}} + + +def test_collect_warns_when_a_schema_is_missing(caplog): + class Page: + title = "Category:Category" + + def get_slot_content(self, slot): + return None + + with caplog.at_level("WARNING", logger="osw.core"): + titles = collect_category_template_titles( + lambda ts: [Page()], ["Category:Category"] + ) + + assert titles == ["Category:Category"] + assert "No jsonschema found for Category:Category" in caplog.text + + +def test_collect_follows_properties_and_items_in_wiki_order(): + meta = "Category:OSW0a1b2c3d4e5f4a6b8c7d9e0f1a2b3c4d" + prop_cat = "Category:OSW11111111222243338444555555555555" + items_cat = "Category:OSW66666666777748889999aaaaaaaaaaaa" + + def ref(title): + return {"$ref": f"/wiki/{title}?action=raw&slot=jsonschema"} + + schemas = { + meta: { + "uuid": "0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d", + "allOf": [ref(CHARACTERISTIC_TYPE)], + "properties": { + "first": ref(prop_cat), + "list": {"type": "array", "items": ref(items_cat)}, + # the second reference to a bundled schema is not walked again + "again": ref(prop_cat), + }, + }, + CHARACTERISTIC_TYPE: {"uuid": "ffe74f29-1d35-4037-b318-c422591c5023"}, + prop_cat: {"uuid": "11111111-2222-4333-8444-555555555555"}, + items_cat: {"uuid": "66666666-7777-4888-9999-aaaaaaaaaaaa"}, + } + requested = [] + + class Page: + def __init__(self, title): + self.title = title + + def get_slot_content(self, slot): + return schemas[self.title] + + def get_pages(titles): + requested.extend(titles) + return [Page(t) for t in titles] + + titles = collect_category_template_titles(get_pages, [meta]) + + # allOf first, then properties in their order, then the uuid of the node itself + assert titles == [ + "Category:Category", + CHARACTERISTIC_TYPE, + prop_cat, + items_cat, + meta, + ] + assert requested.count(prop_cat) == 1 diff --git a/tests/test_store_entity_failure.py b/tests/test_store_entity_failure.py index 6fb03775..163ccd24 100644 --- a/tests/test_store_entity_failure.py +++ b/tests/test_store_entity_failure.py @@ -8,7 +8,7 @@ import pytest import osw.model.entity as model -from osw.core import OSW +from osw.core import OSW, AddOverwriteClassOptions from osw.utils.wiki import get_namespace, get_title from osw.wtsite import WtPage @@ -90,6 +90,37 @@ def test_store_entity_all_success_returns_result(offline_osw, monkeypatch): assert result.failed == {} +def test_partial_error_does_not_count_a_skipped_page_as_stored( + offline_osw, monkeypatch +): + """'keep existing' leaves a page untouched, so it was not stored.""" + keep, ok, boom = ( + model.Item(label=[model.Label(text=text)]) for text in ("Keep", "Ok", "Boom") + ) + keep_title, ok_title, boom_title = (_title(it) for it in (keep, ok, boom)) + # only the kept entity is already on the wiki; the other two are created + monkeypatch.setattr( + WtPage, "init", lambda self: setattr(self, "exists", self.title == keep_title) + ) + _install_edit(monkeypatch, {boom_title}) + + with pytest.raises(OSW.StoreEntityPartialError) as exc_info: + offline_osw.store_entity( + OSW.StoreEntityParam( + entities=[keep, ok, boom], + overwrite=AddOverwriteClassOptions.keep_existing, + parallel=False, + ) + ) + + err = exc_info.value + assert err.stored == [ok_title] + assert err.skipped == [keep_title] + assert set(err.failed) == {boom_title} + # the kept page is still on the wiki, so it stays in 'pages' + assert keep_title in err.result.pages + + def test_store_entity_serial_all_success_returns_result(offline_osw, monkeypatch): items = [model.Item(label=[model.Label(text=f"Ser{i}")]) for i in range(2)] titles = [_title(it) for it in items] diff --git a/tests/test_store_entity_keeps_jsonschema.py b/tests/test_store_entity_keeps_jsonschema.py new file mode 100644 index 00000000..877eb6f2 --- /dev/null +++ b/tests/test_store_entity_keeps_jsonschema.py @@ -0,0 +1,213 @@ +"""Unit tests for the jsonschema slot regeneration in store_entity(). + +Regression guard for two defects in the schema aggregation of store_entity(): + +1. store_entity() passed the stored schema as `schema=`, a field that + AggregateGeneratedSchemasParam does not have. Pydantic ignored it, so the + aggregation always started from `{}` and every save dropped the hand-written + part of the jsonschema slot. +2. In DEFINITIONS_SECTION mode the new generated schema was merged into the + existing `$defs/generated` instead of replacing it, so a `required` entry or a + `type` from an earlier save survived every regeneration. + +These run fully offline: store_entity() runs with offline=True, the meta category +is served by a fake site and the overwrite policy is wrapped to put a stored +jsonschema slot on the page, so no network is required. +""" + +import json + +import pytest +from opensemantic.core.v1 import CharacteristicType + +import osw.model.entity as model +from osw.core import OSW +from osw.wtsite import WtPage + +META_UUID = "0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d" +META_CATEGORY = "Category:OSW0a1b2c3d4e5f4a6b8c7d9e0f1a2b3c4d" +CHARACTERISTIC_TYPE = "Category:OSWffe74f291d354037b318c422591c5023" +GEN_POINTER = "#/$defs/generated" + + +class _FakeMetaCategoryPage: + def __init__(self, title, template, schema): + self.title = title + self._slots = { + "schema_template": template, + "jsondata": {}, + "jsonschema": schema, + } + + def get_slot_content(self, slot_key): + return self._slots[slot_key] + + +class _FakeGetPageResult: + def __init__(self, pages): + self.pages = pages + + +class _FakeSite: + """Serves a meta category with the template under test. + + Like in the wiki, the templates follow the allOf chain of the meta category + schema, so the meta category refers to CharacteristicType when `definitions` + is set. + """ + + def __init__(self, template, definitions): + self.template = template + self.definitions = definitions + + def _page(self, title): + if title == META_CATEGORY: + schema = {"uuid": META_UUID} + if self.definitions: + schema["allOf"] = [ + {"$ref": f"/wiki/{CHARACTERISTIC_TYPE}?action=raw&slot=jsonschema"} + ] + return _FakeMetaCategoryPage(title, self.template, schema) + if title == CHARACTERISTIC_TYPE: + return _FakeMetaCategoryPage( + title, None, {"uuid": "ffe74f29-1d35-4037-b318-c422591c5023"} + ) + return _FakeMetaCategoryPage(title, None, {}) + + def get_page(self, param): + return _FakeGetPageResult([self._page(t) for t in param.titles]) + + +@pytest.fixture +def store(monkeypatch): + """Runs store_entity() offline on a page whose jsonschema slot holds `stored`. + + Returns the jsonschema slot that store_entity() wrote. + """ + original_policy = OSW._apply_overwrite_policy + + def _store(entity, template, stored, definitions=None): + if definitions is None: + # characteristics select the definitions section mode in the wiki + definitions = isinstance(entity, CharacteristicType) + + def policy_with_stored_schema(param): + if stored is not None: + param.page.set_slot_content( + "jsonschema", json.loads(json.dumps(stored)) + ) + return original_policy(param) + + monkeypatch.setattr( + OSW, "_apply_overwrite_policy", staticmethod(policy_with_stored_schema) + ) + osw_obj = OSW.construct(site=_FakeSite(template, definitions)) + result = osw_obj.store_entity( + OSW.StoreEntityParam( + entities=[entity], + namespace="Category", + meta_category_title=META_CATEGORY, + offline=True, + ) + ) + (page,) = result.pages.values() + assert isinstance(page, WtPage) + return page.get_slot_content("jsonschema") + + return _store + + +def _characteristic(): + return CharacteristicType( + uuid="efad8086-4a76-47ca-b278-f5f944e5754b", + name="TestCharacteristic", + label=[{"text": "Test Characteristic"}], + # the default resolves a superclass from a wiki, and oold 2 rejects an + # empty list for this required field + subclass_of=[CHARACTERISTIC_TYPE], + ) + + +def _category(): + # store_entity() selects ROOT_LEVEL mode for any entity that is not a + # characteristic, so a plain Item stored in the Category namespace suffices + return model.Item( + uuid="0d0c1c35-4c4d-4e3a-9c9a-6f4a1b2c3d4e", + name="TestCategory", + label=[model.Label(text="Test Category")], + ) + + +def test_root_level_keeps_hand_written_schema(store): + stored = {"title": "hand", "properties": {"manual": {"type": "string"}}} + template = '{"title": "{{name}}", "properties": {"gen": {"type": "number"}}}' + + schema = store(_category(), template, stored) + + assert schema["properties"]["manual"] == {"type": "string"} + assert schema["properties"]["gen"] == {"type": "number"} + assert schema["title"] == "TestCategory" + + +def test_definitions_section_keeps_hand_written_schema(store): + stored = {"title": "hand", "$defs": {"manual": {"type": "string"}}} + template = '{"title": "{{name}}", "type": "object"}' + + schema = store(_characteristic(), template, stored) + + assert schema["$defs"]["manual"] == {"type": "string"} + assert schema["$defs"]["generated"]["type"] == "object" + assert schema["allOf"] == [{"$ref": GEN_POINTER}] + + +def test_definitions_section_replaces_stale_generated_section(store): + stored = {"title": "hand", "$defs": {"manual": {"type": "string"}}} + first_template = ( + '{"title": "{{name}}", "type": "object", "required": ["type"],' + ' "properties": {"type": {"type": "string"}}}' + ) + second_template = ( + '{"title": "{{name}}", "properties": {"value": {"type": "number"}}}' + ) + + first = store(_characteristic(), first_template, stored) + assert first["$defs"]["generated"]["required"] == ["type"] + + second = store(_characteristic(), second_template, first) + + generated = second["$defs"]["generated"] + assert "required" not in generated + assert "type" not in generated + assert "type" not in generated["properties"] + assert generated["properties"]["value"] == {"type": "number"} + assert generated["$comment"].count(META_CATEGORY) == 1 + assert second["allOf"] == [{"$ref": GEN_POINTER}] + assert second["$defs"]["manual"] == {"type": "string"} + + +def test_failed_generation_keeps_generated_section(store): + stored = { + "title": "hand", + "allOf": [{"$ref": GEN_POINTER}], + "$defs": {"generated": {"type": "object", "required": ["type"]}}, + } + # not valid JSON, so generation fails and produces no generated schema + template = '{"title": {{name}}' + + schema = store(_characteristic(), template, stored) + + assert schema == stored + + +def test_root_level_second_save_does_not_duplicate(store): + stored = {"title": "hand", "properties": {"manual": {"type": "string"}}} + template = ( + '{"title": "{{name}}", "required": ["gen"],' + ' "properties": {"gen": {"type": "number"}}}' + ) + + first = store(_category(), template, stored) + second = store(_category(), template, first) + + assert second == first + assert second["required"] == ["gen"] diff --git a/tests/test_wiki_file_session.py b/tests/test_wiki_file_session.py new file mode 100644 index 00000000..9413114b --- /dev/null +++ b/tests/test_wiki_file_session.py @@ -0,0 +1,203 @@ +"""Unit tests for how a wiki file controller finds its session. + +The session is optional: a controller built from a file entity connects on +first use. A file that is about to be uploaded has no URL yet - that is what +the upload produces - so the URL cannot be the only way to find the wiki. + +Offline: the session is never built here, only the inputs that decide which +one would be. +""" + +from io import BytesIO + +import pytest +from oold.backend.auth import UserPwdCredential, set_credential + +import osw.controller.file.wiki as wiki_module +from osw.controller.file.local import LocalFileController +from osw.controller.file.wiki import OSW, WikiFileController, WtSite + + +@pytest.fixture(autouse=True) +def _clean_credentials(monkeypatch): + """Keep the process-wide credential store and session cache per test.""" + import oold.backend.auth as auth + + saved = auth._credentials.get() + auth._credentials.set({}) + wiki_module.clear_sessions() + yield + auth._credentials.set(saved) + wiki_module.clear_sessions() + + +def _credential(iri, username="bot", password="secret"): + return UserPwdCredential(iri=iri, username=username, password=password) + + +def _a_session() -> OSW: + """A stand-in session: only which one travels is under test. + + pydantic v1 copies a model on assignment, so the controller does not hold + the very object that was passed in; the sessions are told apart by the + site they carry. + """ + return OSW.construct(site=WtSite.__new__(WtSite)) + + +def _controller(**kwargs): + return WikiFileController(**kwargs) + + +# -- choosing the wiki -------------------------------------------------------- + + +def test_a_url_decides_the_wiki(): + controller = _controller(download_url="https://wiki.example.org/wiki/File:A.bin") + assert controller._session_url() == "https://wiki.example.org/wiki/File:A.bin" + + +def test_without_a_url_the_single_registered_credential_decides(): + """A fresh upload has no URL, so the credential is the only pointer.""" + set_credential(_credential("wiki.example.org")) + assert _controller()._session_url() == "wiki.example.org" + + +def test_without_a_url_and_without_a_credential_the_error_says_so(): + with pytest.raises(ValueError, match="no credential is registered"): + _controller()._session_url() + + +def test_several_credentials_without_a_url_are_ambiguous(): + set_credential(_credential("wiki.example.org")) + set_credential(_credential("other.example.org")) + with pytest.raises(ValueError, match=r"other\.example\.org"): + _controller()._session_url() + + +def test_a_session_that_was_passed_in_is_kept(): + sentinel = _a_session() + controller = _controller(osw=sentinel) + assert controller._ensure_session().site is sentinel.site + + +# -- the session cache -------------------------------------------------------- + + +@pytest.fixture +def built_sites(monkeypatch): + """Record the WtSite configs built, instead of connecting to a wiki.""" + configs = [] + + def _config(**kwargs): + configs.append(kwargs) + return kwargs + + monkeypatch.setattr( + wiki_module, "OSW", lambda site: f"session{len(configs)}:{site['iri']}" + ) + monkeypatch.setattr(wiki_module, "WtSite", lambda config: config) + monkeypatch.setattr(wiki_module.WtSite, "WtSiteConfig", _config, raising=False) + return configs + + +def test_sessions_are_cached_per_credential(built_sites): + """Two path-scoped credentials of one host are two logins.""" + set_credential(_credential("wiki.example.org/a", username="a")) + set_credential(_credential("wiki.example.org/b", username="b")) + + first = wiki_module.session_for("https://wiki.example.org/a/File:X") + second = wiki_module.session_for("https://wiki.example.org/b/File:X") + again = wiki_module.session_for("https://wiki.example.org/a/File:Y") + + assert first != second + assert again == first + # and every one of them talks to the host, not to host/path + assert {config["iri"] for config in built_sites} == {"wiki.example.org"} + + +def test_one_credential_does_not_collapse_two_hosts(built_sites): + """oold matches a stored IRI inside the searched one, so one credential + answers for every subdomain - that must not share a session.""" + set_credential(_credential("example.org")) + + first = wiki_module.session_for("https://wiki-a.example.org/wiki/File:X") + second = wiki_module.session_for("https://wiki-b.example.org/wiki/File:Y") + + assert first != second + assert [config["iri"] for config in built_sites] == [ + "wiki-a.example.org", + "wiki-b.example.org", + ] + + +def test_a_bare_credential_iri_still_names_the_host(built_sites): + """The no-URL path hands the credential key straight to session_for.""" + set_credential(_credential("wiki.example.org/w")) + + wiki_module.session_for(_controller()._session_url()) + + assert built_sites[0]["iri"] == "wiki.example.org" + + +def test_a_replaced_credential_replaces_the_session(built_sites): + """A later set_credential must not be ignored for the process lifetime.""" + set_credential(_credential("wiki.example.org", password="old")) + first = wiki_module.session_for("https://wiki.example.org/File:X") + + set_credential(_credential("wiki.example.org", password="new")) + second = wiki_module.session_for("https://wiki.example.org/File:X") + + assert second != first + assert len(built_sites) == 2 + + +def test_a_missing_credential_is_reported_with_the_host(): + with pytest.raises(ValueError, match=r"wiki\.example\.org"): + wiki_module.session_for("https://wiki.example.org/File:X") + + +# -- from_other --------------------------------------------------------------- + + +def test_from_other_keeps_a_session_the_source_carries(): + sentinel = _a_session() + source = WikiFileController(osw=sentinel, download_url="https://w.example/x") + + target = WikiFileController.from_other(source) + + assert target.osw.site is sentinel.site + + +def test_from_other_takes_the_session_it_is_given(): + sentinel = _a_session() + other = _a_session() + source = WikiFileController(osw=other, download_url="https://w.example/x") + + target = WikiFileController.from_other(source, osw=sentinel) + + assert target.osw.site is sentinel.site + assert target.osw.site is not other.site + + +# -- the local controller ----------------------------------------------------- + + +def test_local_controller_needs_a_file(): + with pytest.raises(ValueError, match="needs a file"): + LocalFileController() + + +def test_local_controller_accepts_path_or_local_path(tmp_path): + target = tmp_path / "a.bin" + target.write_bytes(b"x") + + assert LocalFileController(path=target).path == target + assert LocalFileController(local_path=str(target)).path == target + + +def test_local_controller_round_trips_a_file(tmp_path): + controller = LocalFileController(path=tmp_path / "a.bin") + controller.put(BytesIO(b"payload")) + with controller.get() as handle: + assert handle.read() == b"payload" diff --git a/tests/test_wtsite_create_page_package.py b/tests/test_wtsite_create_page_package.py index d2d3c028..da693f92 100644 --- a/tests/test_wtsite_create_page_package.py +++ b/tests/test_wtsite_create_page_package.py @@ -1,10 +1,16 @@ -"""Unit tests for the PagePackageConfig.clear_content_dir purge flag. +"""Unit tests for the PagePackageConfig.clear_content_dir purge flag, and for +the deterministic ordering of packages.json (#172). Regression guard for #42: WtSite.create_page_package used to unconditionally shutil.rmtree() the content directory. clear_content_dir defaults to True (no behaviour change), but setting it to False must keep any existing content. + +Regression guard for #172: WtSite.create_page_package wrote the "pages" array +of packages.json in an order that varied between runs whenever +config.include_files was True, causing spurious diffs. """ +import json import threading import pytest @@ -27,6 +33,51 @@ def _make_fake_wtsite(): return ws +def _make_file_page(ws, title): + """A File: page whose dump() does not attempt a real file download.""" + page = WtPage(wtSite=ws, title=title, do_init=False) + page.exists = True + page.dump = lambda config, _title=title: package.PagePackagePage( + name=_title.split(":")[-1], namespace="NS_FILE", slots={} + ) + return page + + +def _find_order_sensitive_file_titles(): + """Two File: titles for which list(set([a, b])) != list(set([b, a])) under + the current process's (randomized) string hash seed. Used to reproduce the + hash-order dependent bug without relying on a specific PYTHONHASHSEED.""" + for i in range(1000): + for j in range(i + 1, 1000): + a, b = f"File:F{i}.png", f"File:F{j}.png" + if list({a, b}) != list({b, a}): + return a, b + raise RuntimeError("could not find an order-sensitive pair of file titles") + + +def _make_bundle_and_config(tmp_path, subdir, titles, include_files=True): + target = tmp_path / subdir + target.mkdir(parents=True, exist_ok=True) + bundle = package.PagePackageBundle( + packages={ + "TestPkg": package.PagePackage( + globalID="org.test.TestPkg", + description="test package", + version="0.0.1", + baseURL="https://example.org/", + ) + } + ) + return package.PagePackageConfig( + name="TestPkg", + config_path=target / "packages.json", + content_path=target / "content", + bundle=bundle, + titles=titles, + include_files=include_files, + ) + + def _make_config(tmp_path, clear_content_dir): bundle = package.PagePackageBundle( packages={ @@ -84,3 +135,99 @@ def test_create_page_package_honours_clear_content_dir(tmp_path, clear_content_d assert not marker.exists() else: assert marker.exists() + + +def test_create_page_package_per_page_file_order_is_deterministic(tmp_path): + """Regression test for #172: the order in which a page's file references + are discovered must not change where they end up in packages.json.""" + file_a, file_b = _find_order_sensitive_file_titles() + extra_file = "File:Extra.png" + + def _run(subdir, reversed_refs): + ws = _make_fake_wtsite() + item1 = WtPage(wtSite=ws, title="Item:OSW1", do_init=False) + item2 = WtPage(wtSite=ws, title="Item:OSW2", do_init=False) + order = [file_b, file_a] if reversed_refs else [file_a, file_b] + item1.find_file_page_refs_in_slots = lambda slots=None: list(order) + item2.find_file_page_refs_in_slots = lambda slots=None: [extra_file] + + offline_pages = { + "Item:OSW1": item1, + "Item:OSW2": item2, + file_a: _make_file_page(ws, file_a), + file_b: _make_file_page(ws, file_b), + extra_file: _make_file_page(ws, extra_file), + } + + config = _make_bundle_and_config( + tmp_path, subdir, titles=["Item:OSW1", "Item:OSW2"] + ) + ws.create_page_package( + WtSite.CreatePagePackageParam( + config=config, + offline_pages=offline_pages, + debug=False, + ) + ) + return json.loads(config.config_path.read_text(encoding="utf-8")) + + data_1 = _run("run1", reversed_refs=False) + data_2 = _run("run2", reversed_refs=True) + + names_1 = [p["name"] for p in data_1["packages"]["TestPkg"]["pages"]] + names_2 = [p["name"] for p in data_2["packages"]["TestPkg"]["pages"]] + + assert names_1 == names_2 + + # Item:OSW1's own files sit between its dump and Item:OSW2's dump, and are + # in sorted order. + expected_file_names = sorted([file_a.split(":")[-1], file_b.split(":")[-1]]) + assert names_1[1:3] == expected_file_names + + +def test_create_page_package_shared_file_always_attaches_to_first_page(tmp_path): + """Regression test for #172: a file referenced by several pages must + always be attached to the same (first-configured) page, regardless of the + order in which get_page's parallel fetch happens to complete.""" + shared_file = "File:Shared.png" + + def _run(subdir, reverse_fetch_order): + ws = _make_fake_wtsite() + item1 = WtPage(wtSite=ws, title="Item:OSW1", do_init=False) + item2 = WtPage(wtSite=ws, title="Item:OSW2", do_init=False) + item1.find_file_page_refs_in_slots = lambda slots=None: [shared_file] + item2.find_file_page_refs_in_slots = lambda slots=None: [shared_file] + + pages_by_title = { + "Item:OSW1": item1, + "Item:OSW2": item2, + shared_file: _make_file_page(ws, shared_file), + } + + def fake_get_page(param): + titles = list(param.titles) + if reverse_fetch_order: + titles = list(reversed(titles)) + selected = [pages_by_title[t] for t in titles] + return WtSite.GetPageResult(pages=selected, errors=[]) + + ws.get_page = fake_get_page + + config = _make_bundle_and_config( + tmp_path, subdir, titles=["Item:OSW1", "Item:OSW2"] + ) + ws.create_page_package( + WtSite.CreatePagePackageParam(config=config, debug=False) + ) + return json.loads(config.config_path.read_text(encoding="utf-8")) + + data_normal = _run("normal", reverse_fetch_order=False) + data_reversed = _run("reversed", reverse_fetch_order=True) + + names_normal = [p["name"] for p in data_normal["packages"]["TestPkg"]["pages"]] + names_reversed = [p["name"] for p in data_reversed["packages"]["TestPkg"]["pages"]] + + assert names_normal == names_reversed + # Shared.png must be attached to Item:OSW1 (first in config.titles), even + # when Item:OSW2 is fetched first. + assert names_normal == ["OSW1", "Shared.png", "OSW2"] diff --git a/tests/tools/test_user_sync_build.py b/tests/tools/test_user_sync_build.py new file mode 100644 index 00000000..3040ad43 --- /dev/null +++ b/tests/tools/test_user_sync_build.py @@ -0,0 +1,87 @@ +"""Unit tests for entity building/updating helpers (fakes; no live models).""" + +import uuid +from types import SimpleNamespace + +from osw.tools.user_sync.build import _strip_protected, apply_update +from osw.tools.user_sync.mapping import ProposedUser + + +def _proposed(**kw): + base = dict( + uuid=uuid.uuid4(), + username="u", + first_name="F", + surname="S", + label="F S", + orcid=None, + emails=[], + websites=[], + organizations=[], + ) + base.update(kw) + return ProposedUser(**base) + + +def _entity(): + return SimpleNamespace( + label=[], + first_name="", + surname="", + orcid=None, + email=set(), + website={"https://x"}, + organization=["Item:OSWa"], + employment_contract_status="Item:OSWx", + ) + + +def test_strip_protected_unsets_only_protected(): + entity = _entity() + _strip_protected(entity) + assert entity.employment_contract_status is None + assert entity.organization == ["Item:OSWa"] + + +def test_apply_update_strips_protected_when_in_fields(): + entity = _entity() + apply_update(entity, _proposed(), {"employment_contract_status"}) + assert entity.employment_contract_status is None + + +def test_apply_update_strips_protected_when_not_in_fields(): + entity = _entity() + apply_update(entity, _proposed(surname="New"), {"surname"}) + assert entity.employment_contract_status is None + assert entity.surname == "New" + + +def test_apply_update_unsets_organization_when_empty(): + entity = _entity() + apply_update(entity, _proposed(organizations=[]), {"organizations"}) + assert entity.organization is None + + +def test_apply_update_sets_organization_when_present(): + entity = _entity() + apply_update(entity, _proposed(organizations=["Item:OSWb"]), {"organizations"}) + assert entity.organization == ["Item:OSWb"] + + +def test_apply_update_unsets_website_when_empty(): + entity = _entity() + apply_update(entity, _proposed(websites=[]), {"websites"}) + assert entity.website is None + + +def test_apply_update_sets_website_when_present(): + entity = _entity() + apply_update(entity, _proposed(websites=["https://y"]), {"websites"}) + assert entity.website == {"https://y"} + + +def test_apply_update_sets_username_when_accepted(): + entity = _entity() + entity.username = None + apply_update(entity, _proposed(username="u"), {"username"}) + assert entity.username == "u" diff --git a/tests/tools/test_user_sync_config.py b/tests/tools/test_user_sync_config.py new file mode 100644 index 00000000..b987c907 --- /dev/null +++ b/tests/tools/test_user_sync_config.py @@ -0,0 +1,92 @@ +"""Unit tests for the user-sync configuration and public surface.""" + +from osw.tools.user_sync import ( + ORGANIZATION_CATEGORY, + USER_CATEGORY, + SyncConfig, + SyncReport, + config_from_args, + run_user_sync, +) + + +def test_defaults(): + cfg = SyncConfig() + assert cfg.include_non_orcid is True + assert cfg.create_redirects is True + # Websites and organizations are opt-in (off by default); email is standard. + assert cfg.include_websites is False + assert cfg.link_organizations is False + assert cfg.prune is False # removals are opt-in + assert cfg.enabled_optional_fields() == set() + assert cfg.exclude_system_usernames is True + assert "bot" in cfg.excluded_groups + assert "sysop" in cfg.excluded_groups + assert USER_CATEGORY.startswith("Category:OSW") + assert ORGANIZATION_CATEGORY.startswith("Category:OSW") + + +def test_include_system_flag(): + cfg = config_from_args(["--include-system"]) + assert cfg.exclude_system_usernames is False + + +def test_optional_field_flags(): + cfg = config_from_args(["--with-organizations", "--with-websites"]) + assert cfg.link_organizations is True + assert cfg.include_websites is True + assert cfg.enabled_optional_fields() == {"organizations", "websites"} + + +def test_with_extras_enables_all(): + cfg = config_from_args(["--with-extras"]) + assert cfg.enabled_optional_fields() == {"websites", "organizations"} + + +def test_prune_flag(): + assert config_from_args([]).prune is False + assert config_from_args(["--prune"]).prune is True + + +def test_config_from_args_parses_flags(): + cfg = config_from_args([ + "--domain", + "llm4eln.semos.dev", + "--dry-run", + "--limit", + "5", + "--orcid-only", + ]) + assert cfg.domain == "llm4eln.semos.dev" + assert cfg.dry_run is True + assert cfg.limit == 5 + assert cfg.include_non_orcid is False + assert cfg.include_orcid is True + + +def test_account_scope_defaults_and_mw_only(): + cfg = SyncConfig() + assert cfg.include_orcid is True + assert cfg.include_non_orcid is True # both are standard by default + mw_only = config_from_args(["--mw-only"]) + assert mw_only.include_orcid is False + assert mw_only.include_non_orcid is True + + +def test_report_summary(): + report = SyncReport(created=["Item:OSW1"], updated=["Item:OSW2"]) + assert "created=1" in report.summary() + assert "updated=1" in report.summary() + + +def test_run_user_sync_requires_connection(): + import pytest + + with pytest.raises(ValueError, match="connection"): + run_user_sync(SyncConfig()) + + +def test_include_blocked_flag(): + assert SyncConfig().include_blocked is False + assert config_from_args([]).include_blocked is False + assert config_from_args(["--include-blocked"]).include_blocked is True diff --git a/tests/tools/test_user_sync_existing.py b/tests/tools/test_user_sync_existing.py new file mode 100644 index 00000000..7775da39 --- /dev/null +++ b/tests/tools/test_user_sync_existing.py @@ -0,0 +1,203 @@ +"""Unit tests for loading and indexing existing User items.""" + +from types import SimpleNamespace + +import pytest + +from osw import wiki_tools +from osw.tools.user_sync.existing import ( + ExistingUsers, + adopt_existing_pages, + load_existing_users, +) +from osw.wtsite import WtPage + + +class FakeWtPage(WtPage): + """A WtPage that serves a fixed jsondata slot without a wiki.""" + + def __init__(self, title, jsondata=None): + self.title = title + self._jsondata = jsondata or {} + + def get_slot_content(self, slot, *args, **kwargs): + return self._jsondata + + +def _ask_result(titles, continue_offset=None): + """An SMW ``ask`` response as wiki_tools.semantic_search reads it.""" + result = { + "query": { + "results": { + title: {"fulltext": title, "exists": "1", "printouts": {}} + for title in titles + } + } + } + if continue_offset is not None: + result["query-continue-offset"] = continue_offset + return result + + +class FakeSite: + """Answers ``ask`` with the given responses, one per request, in order.""" + + def __init__(self, responses): + self._responses = list(responses) + self.queries = [] + self.stored = {} + + def get_page(self, param): + pages = [FakeWtPage(t, self.stored.get(t, {"uuid": t})) for t in param.titles] + return SimpleNamespace(pages=pages, errors=[]) + + def api(self, action, **params): + assert action == "ask" + self.queries.append(params["query"]) + return self._responses[len(self.queries) - 1] + + def semantic_search(self, query): + return wiki_tools.semantic_search(self, query) + + +class FakeOsw: + def __init__(self, titles, entities, pages=None, missing=()): + self._entities = entities + self.site = FakeSite(pages or [_ask_result(titles)]) + self.missing = set(missing) + self.loaded = None + + def _get_missing_page_titles(self, titles, confirm_delay_s=5): + assert confirm_delay_s == 0 + return [t for t in titles if t in self.missing] + + def load_entity(self, param): + titles = param.titles + self.loaded = list(titles) + assert set(param.offline_pages) == set(titles) + return [self._entities[t] for t in titles] + + +def test_load_indexes_by_username(): + entities = { + "Item:OSW1": SimpleNamespace(username="0000-0002-6374-9831"), + "Item:OSW2": SimpleNamespace(username="Alice"), + } + osw = FakeOsw(["Item:OSW1", "Item:OSW2"], entities) + existing = load_existing_users(osw) + assert set(existing.by_username) == {"0000-0002-6374-9831", "Alice"} + assert existing.get("Alice") is entities["Item:OSW2"] + assert existing.get("missing") is None + + +def test_load_empty(): + osw = FakeOsw([], {}) + existing = load_existing_users(osw) + assert existing.by_username == {} + + +def test_load_skips_entities_without_username(): + entities = { + "Item:OSW1": SimpleNamespace(username=None), + "Item:OSW2": SimpleNamespace(username="Bob"), + } + osw = FakeOsw(["Item:OSW1", "Item:OSW2"], entities) + existing = load_existing_users(osw) + assert set(existing.by_username) == {"Bob"} + + +def test_load_respects_limit(): + entities = { + "Item:OSW1": SimpleNamespace(username="A"), + "Item:OSW2": SimpleNamespace(username="B"), + } + osw = FakeOsw(["Item:OSW1", "Item:OSW2"], entities) + existing = load_existing_users(osw, limit=1) + assert osw.loaded == ["Item:OSW1"] + assert set(existing.by_username) == {"A"} + + +def test_load_pages_through_all_results_with_the_reported_offset(): + entities = { + "Item:OSW1": SimpleNamespace(username="A"), + "Item:OSW2": SimpleNamespace(username="B"), + "Item:OSW3": SimpleNamespace(username="C"), + } + # the wiki cuts the first page at 2 results and reports offset 2 + pages = [ + _ask_result(["Item:OSW1", "Item:OSW2"], continue_offset=2), + _ask_result(["Item:OSW3"]), + ] + osw = FakeOsw([], entities, pages=pages) + existing = load_existing_users(osw) + assert set(existing.by_username) == {"A", "B", "C"} + assert len(osw.site.queries) == 2 + assert "|offset=0" in osw.site.queries[0] + assert "|offset=2" in osw.site.queries[1] + assert osw.loaded == ["Item:OSW1", "Item:OSW2", "Item:OSW3"] + + +def test_load_raises_when_the_reported_offset_does_not_advance(): + pages = [ + _ask_result(["Item:OSW1"], continue_offset=0), + _ask_result(["Item:OSW1"], continue_offset=0), + ] + osw = FakeOsw([], {}, pages=pages) + with pytest.raises(RuntimeError, match="offset"): + load_existing_users(osw) + assert len(osw.site.queries) == 1 + + +def _proposed(username, uid): + return SimpleNamespace( + username=username, + uuid=uid, + full_page_title=f"Item:OSW{uid}", + ) + + +def test_adopt_loads_a_page_that_exists_at_the_deterministic_title(): + found = SimpleNamespace(username=None, uuid="1") + osw = FakeOsw( + [], {"Item:OSW1": found}, pages=[_ask_result([])], missing={"Item:OSW2"} + ) + existing = ExistingUsers() + adopted = adopt_existing_pages( + osw, existing, [_proposed("Alice", "1"), _proposed("Bob", "2")] + ) + assert adopted == ["Item:OSW1"] + assert osw.loaded == ["Item:OSW1"] + assert existing.get("Alice") is found + assert existing.get("Bob") is None + + +def test_adopt_skips_users_already_matched_by_username(): + osw = FakeOsw([], {}, pages=[_ask_result([])]) + existing = ExistingUsers(by_username={"Alice": SimpleNamespace(username="Alice")}) + assert adopt_existing_pages(osw, existing, [_proposed("Alice", "1")]) == [] + assert osw.loaded is None + + +def test_load_records_the_stored_jsondata_per_username(): + entities = { + "Item:OSW1": SimpleNamespace(username="Alice", uuid="u1"), + "Item:OSW2": SimpleNamespace(username="Bob", uuid="u2"), + } + osw = FakeOsw(["Item:OSW1", "Item:OSW2"], entities) + osw.site.stored = { + "Item:OSW1": {"uuid": "u1", "username": "Alice", "website": []}, + "Item:OSW2": {"uuid": "u2", "username": "Bob"}, + } + existing = load_existing_users(osw) + assert existing.get_stored("Alice") == osw.site.stored["Item:OSW1"] + assert "employment_contract_status" not in existing.get_stored("Bob") + assert existing.get_stored("missing") is None + + +def test_adopt_records_the_stored_jsondata(): + found = SimpleNamespace(username=None, uuid="1") + osw = FakeOsw([], {"Item:OSW1": found}, pages=[_ask_result([])]) + osw.site.stored = {"Item:OSW1": {"uuid": "1", "label": [{"text": "A"}]}} + existing = ExistingUsers() + adopt_existing_pages(osw, existing, [_proposed("Alice", "1")]) + assert existing.get_stored("Alice") == osw.site.stored["Item:OSW1"] diff --git a/tests/tools/test_user_sync_interactive.py b/tests/tools/test_user_sync_interactive.py new file mode 100644 index 00000000..ba0a47e9 --- /dev/null +++ b/tests/tools/test_user_sync_interactive.py @@ -0,0 +1,232 @@ +"""Unit tests for the interactive preview and conflict resolution.""" + +import uuid + +from osw.tools.user_sync.interactive import ( + Prompter, + nonempty_fields, + render_preview, + resolve_plan, +) +from osw.tools.user_sync.mapping import ProposedUser +from osw.tools.user_sync.reconcile import ( + CONFLICT, + GAP_FILL, + NEW, + REMOVE, + UNCHANGED, + FieldDiff, + ReconcilePlan, + UserChange, +) + + +class FakeIO: + def __init__(self, answers): + self.answers = list(answers) + self.outputs = [] + + def input(self, prompt=""): + return self.answers.pop(0) + + def output(self, text=""): + self.outputs.append(text) + + +def _proposed(username="u", **kw): + base = dict( + uuid=uuid.uuid4(), + username=username, + first_name="F", + surname="S", + label="F S", + orcid=None, + emails=[], + websites=[], + organizations=[], + ) + base.update(kw) + return ProposedUser(**base) + + +def _change(category, username="u", diffs=None, placeholder=False): + proposed = _proposed(username=username) + proposed.placeholder_name = placeholder + return UserChange( + proposed=proposed, + existing=None if category == NEW else object(), + category=category, + diffs=diffs or [], + ) + + +def _prompter(answers): + io = FakeIO(answers) + return Prompter(io.input, io.output), io + + +def test_render_preview(): + plan = ReconcilePlan(changes=[_change(NEW, "alice", placeholder=True)]) + lines = render_preview(plan) + assert lines[0].startswith("Users: 1") + assert "new=1" in lines[0] + assert "+ alice new" in lines[1] + assert "[placeholder name]" in lines[1] + + +def test_nonempty_fields(): + proposed = _proposed(orcid="https://orcid.org/x", emails=["e@x.org"]) + assert nonempty_fields(proposed) == { + "label", + "first_name", + "surname", + "orcid", + "emails", + } + + +def _mixed_plan(): + return ReconcilePlan( + changes=[ + _change(NEW, "new1"), + _change(GAP_FILL, "gap1", [FieldDiff("emails", set(), {"e"}, GAP_FILL)]), + _change(CONFLICT, "con1", [FieldDiff("surname", "Old", "New", CONFLICT)]), + _change( + CONFLICT, + "mix1", + [ + FieldDiff("emails", set(), {"e"}, GAP_FILL), + FieldDiff("surname", "Old", "New", CONFLICT), + ], + ), + _change(UNCHANGED, "same1"), + ] + ) + + +def test_resolve_keep_existing(): + prompter, _ = _prompter(["k", "y"]) + res = resolve_plan(_mixed_plan(), prompter) + assert [r.change.proposed.username for r in res.creates()] == ["new1"] + updates = {r.change.proposed.username: r.apply_fields for r in res.updates()} + assert updates == {"gap1": {"emails"}, "mix1": {"emails"}} + skipped = {r.change.proposed.username for r in res.resolved if r.action == "skip"} + assert skipped == {"con1", "same1"} + assert res.accepted is True + + +def test_resolve_apply_all(): + prompter, _ = _prompter(["a", "y"]) + res = resolve_plan(_mixed_plan(), prompter) + updates = {r.change.proposed.username: r.apply_fields for r in res.updates()} + assert updates["con1"] == {"surname"} + assert updates["mix1"] == {"emails", "surname"} + + +def test_resolve_abort(): + prompter, _ = _prompter(["x"]) + res = resolve_plan(_mixed_plan(), prompter) + assert res.resolved == [] + assert res.accepted is False + + +def test_resolve_dry_run_makes_no_writes_and_no_prompts(): + prompter, io = _prompter([]) # no answers available + res = resolve_plan(_mixed_plan(), prompter, dry_run=True) + assert res.accepted is False + assert io.answers == [] # nothing was asked + + +def test_resolve_auto_apply_keeps_conflicts(): + prompter, io = _prompter([]) + res = resolve_plan(_mixed_plan(), prompter, auto_apply=True) + assert res.accepted is True + updates = {r.change.proposed.username: r.apply_fields for r in res.updates()} + assert updates == {"gap1": {"emails"}, "mix1": {"emails"}} + assert io.answers == [] + + +def test_resolve_review_field_by_field(): + plan = ReconcilePlan( + changes=[ + _change( + CONFLICT, + "con1", + [ + FieldDiff("surname", "Old", "New", CONFLICT), + FieldDiff("label", "L1", "L2", CONFLICT), + ], + ) + ] + ) + prompter, _ = _prompter(["r", "f", "n", "k", "y"]) + res = resolve_plan(plan, prompter) + assert res.updates()[0].apply_fields == {"surname"} + assert res.accepted is True + + +def test_resolve_review_take_all_for_user(): + plan = ReconcilePlan( + changes=[ + _change( + CONFLICT, + "con1", + [ + FieldDiff("surname", "Old", "New", CONFLICT), + FieldDiff("label", "L1", "L2", CONFLICT), + ], + ) + ] + ) + prompter, _ = _prompter(["r", "a", "y"]) + res = resolve_plan(plan, prompter) + assert res.updates()[0].apply_fields == {"surname", "label"} + + +def test_final_confirm_no(): + plan = ReconcilePlan(changes=[_change(NEW, "new1")]) + prompter, _ = _prompter(["n"]) + res = resolve_plan(plan, prompter) + assert res.accepted is False + assert res.creates()[0].change.proposed.username == "new1" + + +def test_removals_auto_apply_and_marked_in_preview(): + plan = ReconcilePlan( + changes=[ + _change( + GAP_FILL, + "u1", + [FieldDiff("organizations", {"x"}, None, REMOVE)], + ) + ] + ) + assert any("-organizations" in ln for ln in render_preview(plan)) + prompter, _ = _prompter(["y"]) # only the final confirm + res = resolve_plan(plan, prompter) + assert res.updates()[0].apply_fields == {"organizations"} + assert res.accepted is True + + +def test_conflict_user_still_auto_applies_removal(): + plan = ReconcilePlan( + changes=[ + _change( + CONFLICT, + "u1", + [ + FieldDiff("surname", "Old", "New", CONFLICT), + FieldDiff("organizations", {"x"}, None, REMOVE), + ], + ) + ] + ) + prompter, _ = _prompter(["k", "y"]) # keep existing on conflict, final yes + res = resolve_plan(plan, prompter) + assert res.updates()[0].apply_fields == {"organizations"} + + +def test_prompter_reprompts_on_invalid(): + prompter, io = _prompter(["z", "k"]) + assert prompter.ask("q", ("k", "n")) == "k" + assert any("Please choose one of" in o for o in io.outputs) diff --git a/tests/tools/test_user_sync_mapping.py b/tests/tools/test_user_sync_mapping.py new file mode 100644 index 00000000..0b6c72f5 --- /dev/null +++ b/tests/tools/test_user_sync_mapping.py @@ -0,0 +1,128 @@ +"""Unit tests for mapping sources to proposed User/Organization items.""" + +import uuid + +from osw.tools.user_sync import mapping +from osw.tools.user_sync.mapping import ( + ProposedUser, + map_organization, + map_user, + normalize_org_name, + organization_uuid, + user_uuid, +) +from osw.tools.user_sync.sources import MwUser, OrcidAffiliation, OrcidProfile + +ORCID = "0000-0002-6374-9831" + + +def _profile(**kw): + base = dict(orcid_id=ORCID, given_names="Lukas", family_name="Koschmieder") + base.update(kw) + return OrcidProfile(**base) + + +def test_map_orcid_user_full_profile(): + mw = MwUser(name=ORCID) + profile = _profile( + emails=["l@example.org"], + urls=["https://example.org"], + affiliations=[OrcidAffiliation(organization="Example University")], + ) + proposed, orgs = map_user( + mw, profile, include_websites=True, link_organizations=True + ) + assert isinstance(proposed, ProposedUser) + assert proposed.username == ORCID + assert proposed.orcid == f"https://orcid.org/{ORCID}" + assert proposed.first_name == "Lukas" + assert proposed.surname == "Koschmieder" + assert proposed.label == "Lukas Koschmieder" + assert proposed.emails == ["l@example.org"] + assert proposed.websites == ["https://example.org"] + assert proposed.placeholder_name is False + assert len(orgs) == 1 + assert proposed.organizations == [orgs[0].full_page_title] + assert proposed.full_page_title.startswith("Item:OSW") + + +def test_map_orcid_user_without_profile_uses_placeholder_names(): + mw = MwUser(name=ORCID) + proposed, orgs = map_user(mw, None) + assert proposed.orcid == f"https://orcid.org/{ORCID}" + assert proposed.first_name == ORCID + assert proposed.surname == ORCID + assert proposed.placeholder_name is True + assert orgs == [] + + +def test_map_non_orcid_user(): + mw = MwUser(name="Alice") + proposed, _orgs = map_user(mw, None) + assert proposed.orcid is None + assert proposed.placeholder_name is True + assert proposed.uuid == uuid.uuid5(mapping.USER_NAMESPACE, "Alice") + + +def test_user_uuid_is_deterministic_and_orcid_keyed(): + mw = MwUser(name=ORCID) + expected = uuid.uuid5(mapping.USER_NAMESPACE, ORCID) + assert user_uuid(mw) == expected + assert user_uuid(mw) == user_uuid(MwUser(name=ORCID)) + + +def test_credit_name_split_when_no_given_family(): + mw = MwUser(name=ORCID) + profile = OrcidProfile( + orcid_id=ORCID, given_names=None, family_name=None, credit_name="Jane A. Doe" + ) + proposed, _ = map_user(mw, profile) + assert proposed.first_name == "Jane" + assert proposed.surname == "Doe" + assert proposed.label == "Jane A. Doe" + assert proposed.placeholder_name is False + + +def test_organization_uuid_prefers_ror_over_name(): + ror = OrcidAffiliation(organization="Example U", ror_id="https://ror.org/01") + named = OrcidAffiliation(organization="Example U") + assert organization_uuid(ror) != organization_uuid(named) + assert organization_uuid(named) == uuid.uuid5( + mapping.ORGANIZATION_NAMESPACE, normalize_org_name("Example U") + ) + assert map_organization(ror).ror_id == "https://ror.org/01" + + +def test_duplicate_affiliations_deduped(): + mw = MwUser(name=ORCID) + profile = _profile( + affiliations=[ + OrcidAffiliation(organization="Example University"), + OrcidAffiliation(organization="example university"), + ] + ) + proposed, orgs = map_user(mw, profile, link_organizations=True) + assert len(orgs) == 1 + assert len(proposed.organizations) == 1 + + +def test_link_organizations_disabled(): + mw = MwUser(name=ORCID) + profile = _profile(affiliations=[OrcidAffiliation(organization="Example U")]) + proposed, orgs = map_user(mw, profile, link_organizations=False) + assert orgs == [] + assert proposed.organizations == [] + + +def test_email_always_included_but_extras_gated_by_default(): + mw = MwUser(name=ORCID) + profile = _profile( + emails=["l@example.org"], + urls=["https://example.org"], + affiliations=[OrcidAffiliation(organization="Example U")], + ) + proposed, orgs = map_user(mw, profile) # defaults: websites/orgs off + assert proposed.emails == ["l@example.org"] + assert proposed.websites == [] + assert proposed.organizations == [] + assert orgs == [] diff --git a/tests/tools/test_user_sync_orcid.py b/tests/tools/test_user_sync_orcid.py new file mode 100644 index 00000000..158a3db2 --- /dev/null +++ b/tests/tools/test_user_sync_orcid.py @@ -0,0 +1,147 @@ +"""Unit tests for the ORCID public-API client.""" + +import pytest +import requests + +from osw.tools.user_sync import sources +from osw.tools.user_sync.sources import ( + OrcidProfile, + OrcidRateLimitError, + fetch_orcid_record, + parse_orcid_record, +) + +ORCID_ID = "0000-0002-1825-0097" + +RECORD = { + "person": { + "name": { + "given-names": {"value": "Jane"}, + "family-name": {"value": "Doe"}, + "credit-name": {"value": "Jane A. Doe"}, + }, + "other-names": {"other-name": [{"content": "J. Doe"}]}, + "emails": {"email": [{"email": "jane@example.org"}]}, + "researcher-urls": { + "researcher-url": [{"url": {"value": "https://jane.example.org"}}] + }, + }, + "activities-summary": { + "employments": { + "affiliation-group": [ + { + "summaries": [ + { + "employment-summary": { + "department-name": "Physics", + "role-title": "Researcher", + "organization": { + "name": "Example University", + "disambiguated-organization": { + "disambiguated-organization-identifier": ( + "https://ror.org/01abc2345" + ), + "disambiguation-source": "ROR", + }, + }, + } + } + ] + } + ] + } + }, +} + + +class FakeResponse: + def __init__(self, status_code=200, json_data=None, headers=None): + self.status_code = status_code + self._json = json_data + self.headers = headers or {} + + def json(self): + return self._json + + def raise_for_status(self): + if self.status_code >= 400: + raise requests.HTTPError(f"status {self.status_code}") + + +class FakeSession: + def __init__(self, responses): + self._responses = list(responses) + self.calls = 0 + + def get(self, url, headers=None, timeout=None): + self.calls += 1 + return self._responses.pop(0) + + +def test_parse_record_full(): + p = parse_orcid_record(RECORD, ORCID_ID) + assert p.given_names == "Jane" + assert p.family_name == "Doe" + assert p.credit_name == "Jane A. Doe" + assert p.display_name == "Jane A. Doe" + assert p.other_names == ["J. Doe"] + assert p.emails == ["jane@example.org"] + assert p.urls == ["https://jane.example.org"] + assert p.orcid_uri == f"https://orcid.org/{ORCID_ID}" + assert len(p.affiliations) == 1 + aff = p.affiliations[0] + assert aff.organization == "Example University" + assert aff.ror_id == "https://ror.org/01abc2345" + assert aff.department == "Physics" + assert aff.role == "Researcher" + + +def test_parse_record_sparse(): + p = parse_orcid_record({"person": {}}, ORCID_ID) + assert p.given_names is None + assert p.display_name is None + assert p.emails == [] + assert p.affiliations == [] + + +def test_display_name_falls_back_to_given_family(): + p = OrcidProfile(orcid_id=ORCID_ID, given_names="Jane", family_name="Doe") + assert p.display_name == "Jane Doe" + + +def test_fetch_404_returns_none(): + session = FakeSession([FakeResponse(status_code=404)]) + assert fetch_orcid_record(ORCID_ID, session=session) is None + + +def test_fetch_200_parses(): + session = FakeSession([FakeResponse(status_code=200, json_data=RECORD)]) + p = fetch_orcid_record(ORCID_ID, session=session) + assert p.family_name == "Doe" + + +def test_fetch_retries_on_429(monkeypatch): + monkeypatch.setattr(sources.time, "sleep", lambda *_: None) + session = FakeSession([ + FakeResponse(status_code=429, headers={"Retry-After": "0"}), + FakeResponse(status_code=200, json_data=RECORD), + ]) + p = fetch_orcid_record(ORCID_ID, session=session) + assert p.given_names == "Jane" + assert session.calls == 2 + + +def test_fetch_raises_when_rate_limited(monkeypatch): + monkeypatch.setattr(sources.time, "sleep", lambda *_: None) + session = FakeSession([FakeResponse(status_code=429) for _ in range(3)]) + with pytest.raises(OrcidRateLimitError): + fetch_orcid_record(ORCID_ID, session=session, max_retries=2) + + +def test_fetch_uses_cache(): + session = FakeSession([FakeResponse(status_code=200, json_data=RECORD)]) + cache = {} + first = fetch_orcid_record(ORCID_ID, session=session, cache=cache) + second = fetch_orcid_record(ORCID_ID, session=session, cache=cache) + assert first is second + assert session.calls == 1 diff --git a/tests/tools/test_user_sync_reconcile.py b/tests/tools/test_user_sync_reconcile.py new file mode 100644 index 00000000..d49ef8a7 --- /dev/null +++ b/tests/tools/test_user_sync_reconcile.py @@ -0,0 +1,287 @@ +"""Unit tests for the reconciliation engine.""" + +import uuid +from types import SimpleNamespace + +from osw.tools.user_sync.existing import ExistingUsers +from osw.tools.user_sync.mapping import ProposedUser +from osw.tools.user_sync.reconcile import ( + CONFLICT, + GAP_FILL, + NEW, + REMOVE, + UNCHANGED, + reconcile, + reconcile_user, +) + +ORCID = "0000-0002-6374-9831" + + +def _proposed(**kw): + base = dict( + uuid=uuid.uuid4(), + username=ORCID, + first_name="Lukas", + surname="Koschmieder", + label="Lukas Koschmieder", + orcid=f"https://orcid.org/{ORCID}", + emails=[], + websites=[], + organizations=[], + ) + base.update(kw) + return ProposedUser(**base) + + +def _entity(**kw): + base = dict( + username=ORCID, + label="Lukas Koschmieder", + first_name="Lukas", + surname="Koschmieder", + orcid=f"https://orcid.org/{ORCID}", + email=None, + website=None, + organization=None, + ) + base.update(kw) + return SimpleNamespace( + username=base["username"], + label=[SimpleNamespace(text=base["label"])] if base["label"] else [], + first_name=base["first_name"], + surname=base["surname"], + orcid=base["orcid"], + email=set(base["email"] or []), + website=set(base["website"] or []), + organization=set(base["organization"] or []), + ) + + +def test_new_when_no_existing(): + change = reconcile_user(_proposed(), None) + assert change.category == NEW + assert change.diffs == [] + + +def test_unchanged_when_equal(): + change = reconcile_user(_proposed(), _entity()) + assert change.category == UNCHANGED + assert change.diffs == [] + + +def test_gap_fill_for_missing_field(): + proposed = _proposed(emails=["l@example.org"]) + change = reconcile_user(proposed, _entity(email=None)) + assert change.category == GAP_FILL + assert [d.name for d in change.diffs] == ["emails"] + assert change.diffs[0].status == GAP_FILL + + +def test_conflict_on_differing_value(): + change = reconcile_user(_proposed(surname="Koschmieder"), _entity(surname="Kosch")) + assert change.category == CONFLICT + assert change.diffs[0].name == "surname" + assert change.diffs[0].existing == "Kosch" + assert change.diffs[0].proposed == "Koschmieder" + + +def test_conflict_dominates_gap_fill(): + proposed = _proposed(surname="Koschmieder", emails=["l@example.org"]) + change = reconcile_user(proposed, _entity(surname="Kosch", email=None)) + assert change.category == CONFLICT + names = {d.name for d in change.diffs} + assert names == {"surname", "emails"} + + +def test_empty_proposal_does_not_clear_existing(): + change = reconcile_user(_proposed(emails=[]), _entity(email=["keep@example.org"])) + assert change.category == UNCHANGED + assert change.diffs == [] + + +def test_orcid_trailing_slash_not_a_conflict(): + change = reconcile_user( + _proposed(orcid=f"https://orcid.org/{ORCID}"), + _entity(orcid=f"https://orcid.org/{ORCID}/"), + ) + assert change.category == UNCHANGED + + +def test_set_order_independent(): + change = reconcile_user( + _proposed(organizations=["Item:OSWb", "Item:OSWa"]), + _entity(organization=["Item:OSWa", "Item:OSWb"]), + enabled_fields=frozenset({"organizations"}), + ) + assert change.category == UNCHANGED + + +def test_disabled_optional_field_removed_when_pruning(): + # organizations disabled + existing value + prune -> REMOVE + change = reconcile_user( + _proposed(), _entity(organization=["Item:OSWa"]), prune=True + ) + assert change.category == GAP_FILL + removes = [d for d in change.diffs if d.status == REMOVE] + assert [d.name for d in removes] == ["organizations"] + assert removes[0].proposed is None + + +def test_prune_off_keeps_disabled_and_protected(): + entity = _entity() + entity.__iris__ = { + "organization": ["Item:OSWa"], + "employment_contract_status": "Item:OSWx", + } + change = reconcile_user(_proposed(), entity) # prune off (default) + assert change.category == UNCHANGED + assert change.diffs == [] + + +def test_enabled_optional_field_not_removed(): + change = reconcile_user( + _proposed(organizations=["Item:OSWa"]), + _entity(organization=["Item:OSWa"]), + enabled_fields=frozenset({"organizations"}), + ) + assert change.category == UNCHANGED + + +def test_protected_field_removed_when_pruning(): + entity = _entity() + entity.__iris__ = {"employment_contract_status": "Item:OSWx"} + change = reconcile_user(_proposed(), entity, prune=True) + removes = [d.name for d in change.diffs if d.status == REMOVE] + assert "employment_contract_status" in removes + assert change.category == GAP_FILL + + +def test_protected_default_not_stored_is_not_removed(): + # The loaded entity shows the model default IRI, the stored page has none. + entity = _entity() + entity.__iris__ = {"employment_contract_status": "Item:OSWdefault"} + change = reconcile_user( + _proposed(), entity, prune=True, stored_jsondata={"username": ORCID} + ) + assert [d for d in change.diffs if d.status == REMOVE] == [] + assert change.category == UNCHANGED + + +def test_protected_field_stored_is_removed_when_pruning(): + entity = _entity() + entity.__iris__ = {"employment_contract_status": "Item:OSWx"} + change = reconcile_user( + _proposed(), + entity, + prune=True, + stored_jsondata={"employment_contract_status": "Item:OSWx"}, + ) + removes = [d.name for d in change.diffs if d.status == REMOVE] + assert removes == ["employment_contract_status"] + + +def test_reconcile_uses_the_stored_jsondata_of_the_index(): + entity = _entity() + entity.__iris__ = {"employment_contract_status": "Item:OSWdefault"} + existing = ExistingUsers( + by_username={ORCID: entity}, stored_jsondata={ORCID: {"username": ORCID}} + ) + plan = reconcile([_proposed()], existing, prune=True) + assert plan.changes[0].category == UNCHANGED + + +def test_missing_label_is_gap_fill(): + change = reconcile_user(_proposed(label="Lukas Koschmieder"), _entity(label=None)) + assert change.category == GAP_FILL + assert change.diffs[0].name == "label" + + +class _OrgTrap: + """A loaded-entity stand-in whose organization attribute must never be read.""" + + __iris__ = {"organization": ["Item:OSWa", "Item:OSWb"]} + label = [SimpleNamespace(text="X Y")] + first_name = "X" + surname = "Y" + orcid = None + email: set = set() + website: set = set() + + @property + def organization(self): + raise RuntimeError("relation must not be resolved during reconcile") + + +def test_existing_orgs_read_from_iris_without_resolving(): + from osw.tools.user_sync.reconcile import existing_fields + + ef = existing_fields(_OrgTrap()) + assert ef["organizations"] == {"Item:OSWa", "Item:OSWb"} + + +def test_reconcile_plan_counts(): + proposed = [ + _proposed(username="new-user"), + _proposed(username=ORCID), + _proposed(username="conflict-user", surname="New"), + ] + existing = ExistingUsers( + by_username={ + ORCID: _entity(), + "conflict-user": _entity(surname="Old"), + } + ) + plan = reconcile(proposed, existing) + counts = plan.counts() + assert counts == {NEW: 1, GAP_FILL: 0, CONFLICT: 1, UNCHANGED: 1} + assert [c.proposed.username for c in plan.conflicts] == ["conflict-user"] + + +def test_placeholder_name_is_not_compared_for_an_existing_item(): + proposed = _proposed( + first_name="bob", surname="bob", label="bob", placeholder_name=True + ) + for existing in ( + _entity(label="Robert Smith", first_name="Robert", surname="Smith"), + _entity(label=None, first_name=None, surname=None), + ): + change = reconcile_user(proposed, existing) + names = {d.name for d in change.diffs} + assert not names & {"label", "first_name", "surname"} + assert change.category == UNCHANGED + + +def test_placeholder_name_is_kept_for_a_new_item(): + change = reconcile_user(_proposed(placeholder_name=True), None) + assert change.category == NEW + + +def test_missing_username_is_filled_on_an_existing_item(): + # An item found by its page title instead of its username has none yet. + change = reconcile_user(_proposed(), _entity(username=None)) + assert change.category == GAP_FILL + assert [(d.name, d.proposed) for d in change.diffs] == [("username", ORCID)] + + +def test_placeholder_user_without_username_still_gets_it(): + change = reconcile_user( + _proposed(placeholder_name=True, label=ORCID), _entity(username=None) + ) + assert change.category == GAP_FILL + assert [d.name for d in change.diffs] == ["username"] + + +def test_protected_field_stored_as_empty_list_is_removed_when_pruning(): + # [] is not a valid value for the single-valued field, so it is removed. + entity = _entity() + entity.__iris__ = {"employment_contract_status": "Item:OSWdefault"} + change = reconcile_user( + _proposed(), + entity, + prune=True, + stored_jsondata={"username": ORCID, "employment_contract_status": []}, + ) + assert [(d.name, d.status) for d in change.diffs] == [ + ("employment_contract_status", REMOVE) + ] diff --git a/tests/tools/test_user_sync_redirects.py b/tests/tools/test_user_sync_redirects.py new file mode 100644 index 00000000..1c1f36df --- /dev/null +++ b/tests/tools/test_user_sync_redirects.py @@ -0,0 +1,76 @@ +"""Unit tests for redirect page handling.""" + +from osw.tools.user_sync import redirects +from osw.tools.user_sync.redirects import ( + ensure_redirect, + is_redirect, + redirect_wikitext, +) + + +def test_redirect_wikitext(): + assert redirect_wikitext("Item:OSW123") == "#REDIRECT [[Item:OSW123]]" + + +def test_is_redirect(): + assert is_redirect("#REDIRECT [[Item:OSW1]]") + assert is_redirect(" #redirect [[x]]") + assert not is_redirect("Some real user page content") + + +class FakeWtPage: + exists_map: dict = {} + content_map: dict = {} + + def __init__(self, wtSite=None, title=None, do_init=True): + self.title = title + self.exists = self.exists_map.get(title, False) + self._content = self.content_map.get(title, "") + self.written = None + self.edited = False + + def get_slot_content(self, slot_key): + return self._content + + def set_slot_content(self, slot_key, content): + self.written = content + + def edit(self, comment=None): + self.edited = True + + +class FakeOsw: + site = object() + + +def _patch(monkeypatch, exists_map, content_map): + FakeWtPage.exists_map = exists_map + FakeWtPage.content_map = content_map + monkeypatch.setattr(redirects, "WtPage", FakeWtPage) + + +def test_ensure_redirect_creates_missing(monkeypatch): + _patch(monkeypatch, {}, {}) + written = ensure_redirect(FakeOsw(), "0000-0002-6374-9831", "Item:OSW1") + assert written == "User:0000-0002-6374-9831" + + +def test_ensure_redirect_overwrites_existing_redirect(monkeypatch): + title = "User:alice" + _patch(monkeypatch, {title: True}, {title: "#REDIRECT [[Item:OSWold]]"}) + written = ensure_redirect(FakeOsw(), "alice", "Item:OSWnew") + assert written == title + + +def test_ensure_redirect_skips_real_page(monkeypatch): + title = "User:bob" + _patch(monkeypatch, {title: True}, {title: "This is my real user page."}) + written = ensure_redirect(FakeOsw(), "bob", "Item:OSW1") + assert written is None + + +def test_ensure_redirect_skips_when_already_correct(monkeypatch): + title = "User:carol" + _patch(monkeypatch, {title: True}, {title: "#REDIRECT [[Item:OSW1]]"}) + written = ensure_redirect(FakeOsw(), "carol", "Item:OSW1") + assert written is None diff --git a/tests/tools/test_user_sync_sources.py b/tests/tools/test_user_sync_sources.py new file mode 100644 index 00000000..758603d0 --- /dev/null +++ b/tests/tools/test_user_sync_sources.py @@ -0,0 +1,171 @@ +"""Unit tests for MediaWiki user enumeration and classification.""" + +from osw.tools.user_sync.sources import ( + MwUser, + enumerate_mw_users, + is_orcid_username, + partition_by_orcid, +) + + +class FakeSite: + """Mimics mwclient Site.api for list=allusers, with continuation.""" + + def __init__(self, pages): + self._pages = pages + self.calls = 0 + + def api(self, action, **params): + assert action == "query" + assert params["list"] == "allusers" + page = self._pages[self.calls] + self.calls += 1 + return page + + +def _page(users, cont=None): + resp = {"query": {"allusers": users}} + if cont is not None: + resp["continue"] = cont + return resp + + +def test_is_orcid_username(): + assert is_orcid_username("0000-0002-6374-9831") + assert is_orcid_username("0000-0003-0930-082X") + assert not is_orcid_username("Alice") + assert not is_orcid_username("") + + +def test_from_api_classifies_orcid_and_blocked(): + u = MwUser.from_api({ + "name": "0000-0002-6374-9831", + "userid": 16, + "registration": "2026-01-21T08:31:14Z", + "editcount": 37, + "groups": ["*", "user", "autoconfirmed"], + }) + assert u.is_orcid is True + assert u.orcid_id == "0000-0002-6374-9831" + assert u.editcount == 37 + assert u.blocked is False + + +def test_enumerate_follows_continuation_and_excludes_bots(): + site = FakeSite([ + _page( + [ + {"name": "0000-0002-6374-9831", "userid": 16, "groups": ["user"]}, + {"name": "SyncBot", "userid": 2, "groups": ["user", "bot"]}, + ], + cont={"aufrom": "M"}, + ), + _page([ + {"name": "Maintainer", "userid": 3, "groups": ["sysop"]}, + ]), + ]) + users = enumerate_mw_users(site) + names = [u.name for u in users] + assert names == ["0000-0002-6374-9831", "Maintainer"] # bot dropped + assert site.calls == 2 # continuation followed + + +def test_enumerate_orcid_only_and_limit(): + site = FakeSite([ + _page([ + {"name": "0000-0002-6374-9831", "groups": ["user"]}, + {"name": "Alice", "groups": ["user"]}, + {"name": "0000-0003-0930-082X", "groups": ["user"]}, + ]) + ]) + orcid_only = enumerate_mw_users(site, include_non_orcid=False) + assert [u.name for u in orcid_only] == [ + "0000-0002-6374-9831", + "0000-0003-0930-082X", + ] + + site2 = FakeSite([ + _page([ + {"name": "0000-0002-6374-9831", "groups": ["user"]}, + {"name": "Alice", "groups": ["user"]}, + ]) + ]) + limited = enumerate_mw_users(site2, limit=1) + assert len(limited) == 1 + + +def test_enumerate_excludes_privileged_groups_and_reserved_names(): + site = FakeSite([ + _page([ + {"name": "0000-0002-6374-9831", "groups": ["user"]}, + {"name": "Admin", "groups": ["user", "sysop", "bureaucrat"]}, + {"name": "Maintenance script", "groups": ["user"]}, + ]) + ]) + users = enumerate_mw_users( + site, + excluded_groups=("bot", "sysop", "bureaucrat"), + excluded_usernames={"Maintenance script"}, + ) + assert [u.name for u in users] == ["0000-0002-6374-9831"] + + +def test_enumerate_keeps_orcid_accounts_despite_excluded_groups(): + site = FakeSite([ + _page([ + {"name": "0000-0001-7620-1245", "groups": ["user", "bot", "sysop"]}, + {"name": "0000-0001-7620-1245@importer", "groups": ["user", "bot"]}, + {"name": "SomeBot", "groups": ["user", "bot"]}, + {"name": "Admin", "groups": ["user", "sysop"]}, + ]) + ]) + users = enumerate_mw_users(site, excluded_groups=("bot", "sysop")) + assert [u.name for u in users] == ["0000-0001-7620-1245"] + + +def test_enumerate_mw_only_skips_orcid(): + site = FakeSite([ + _page([ + {"name": "0000-0002-6374-9831", "groups": ["user"]}, + {"name": "Alice", "groups": ["user"]}, + ]) + ]) + users = enumerate_mw_users(site, include_orcid=False) + assert [u.name for u in users] == ["Alice"] + + +def test_partition_by_orcid(): + users = [ + MwUser(name="0000-0002-6374-9831"), + MwUser(name="Alice"), + ] + parts = partition_by_orcid(users) + assert [u.name for u in parts["orcid"]] == ["0000-0002-6374-9831"] + assert [u.name for u in parts["other"]] == ["Alice"] + + +def test_blocked_users_are_skipped_by_default_and_listed(): + site = FakeSite([ + _page([ + {"name": "Alice", "groups": ["user"]}, + {"name": "Mallory", "groups": ["user"], "blockid": 5}, + {"name": "0000-0002-6374-9831", "groups": ["user"], "blockid": 6}, + ]) + ]) + skipped = [] + users = enumerate_mw_users(site, skipped_blocked=skipped) + assert [u.name for u in users] == ["Alice"] + assert skipped == ["Mallory", "0000-0002-6374-9831"] + + +def test_blocked_users_are_kept_with_exclude_blocked_false(): + site = FakeSite([ + _page([ + {"name": "Alice", "groups": ["user"]}, + {"name": "Mallory", "groups": ["user"], "blockid": 5}, + ]) + ]) + skipped = [] + users = enumerate_mw_users(site, exclude_blocked=False, skipped_blocked=skipped) + assert [u.name for u in users] == ["Alice", "Mallory"] + assert skipped == [] diff --git a/tests/tools/test_user_sync_store.py b/tests/tools/test_user_sync_store.py new file mode 100644 index 00000000..955b7a6c --- /dev/null +++ b/tests/tools/test_user_sync_store.py @@ -0,0 +1,383 @@ +"""Tests for the store paths of the sync and the merge with stored page content.""" + +import json +import uuid +from types import SimpleNamespace +from typing import Any, Dict, Union + +from opensemantic.base.v1 import User + +from osw.core import OSW, OverwriteOptions +from osw.data.import_utility import uuid_to_full_page_title +from osw.tools.user_sync import sync as sync_module +from osw.tools.user_sync.build import apply_update, build_user +from osw.tools.user_sync.config import SyncConfig +from osw.tools.user_sync.interactive import Prompter +from osw.tools.user_sync.mapping import ProposedUser, user_uuid +from osw.tools.user_sync.prune import clear_jsondata_keys +from osw.tools.user_sync.sources import MwUser +from osw.tools.user_sync.sync import run_user_sync +from osw.wtsite import WtPage + +ITEM_TITLE = "Item:OSWb0b1a0a0a0a04a0a8a0a0a0a0a0a0a01" + + +class FakeMwSite: + def __init__(self, users): + self._users = users + + def api(self, action, **params): + return {"query": {"allusers": self._users}} + + +class FakeWtPage(WtPage): + """A WtPage that serves a fixed jsondata slot without a wiki.""" + + def __init__(self, title, jsondata=None): + self.title = title + self._jsondata = jsondata or {} + + def get_slot_content(self, slot, *args, **kwargs): + return self._jsondata + + +class FakeStoreOsw: + """Fake connection whose store_entity returns the stage StoreEntityResult.""" + + def __init__(self, users, existing=None, drop_username=False): + self.mw_site = FakeMwSite(users) + self.site = SimpleNamespace( + semantic_search=self._semantic_search, get_page=self._get_page + ) + self.existing = existing or {} + self.drop_username = drop_username + self.store_params = [] + + def _semantic_search(self, param): + titles = list(self.existing) + return [ + SimpleNamespace( + titles=titles, truncated=False, next_offset=None, count=len(titles) + ) + ] + + def _get_missing_page_titles(self, titles, confirm_delay_s=5): + return [t for t in titles if t not in self.existing] + + def _get_page(self, param): + return SimpleNamespace(pages=[FakeWtPage(t) for t in param.titles], errors=[]) + + def load_entity(self, titles): + titles = getattr(titles, "titles", titles) + if isinstance(titles, str): + titles = [titles] + loaded = [] + for title in titles: + entity = self.existing.get(title) + if entity is None: + name = None if self.drop_username else "loaded" + entity = SimpleNamespace(username=name) + loaded.append(entity) + return loaded + + def store_entity(self, param): + self.store_params.append(param) + return OSW.StoreEntityResult(change_id="c1", pages={}) + + +def _run(osw, **config): + config.setdefault("auto_apply", True) + return run_user_sync( + SyncConfig(domain="d", **config), + osw=osw, + prompter=Prompter(lambda prompt="": "y", lambda text="": None), + session=None, + ) + + +def _patch_redirects(monkeypatch): + written = [] + + def fake_ensure(osw, username, item_title): + written.append((username, item_title)) + return f"User:{username}" + + monkeypatch.setattr(sync_module, "ensure_redirect", fake_ensure) + return written + + +def _title(username): + return uuid_to_full_page_title(user_uuid(MwUser(name=username))) + + +def test_accepted_run_stores_and_writes_redirects(monkeypatch): + written = _patch_redirects(monkeypatch) + osw = FakeStoreOsw([ + {"name": "Alice", "groups": ["user"]}, + {"name": "Bob", "groups": ["user"]}, + ]) + report = _run(osw) + assert report.accepted is True + param = osw.store_params[0] + assert param.overwrite == OverwriteOptions.true + assert param.remove_empty is True + assert report.created == [_title("Alice"), _title("Bob")] + assert report.updated == [] + assert written == [("Alice", _title("Alice")), ("Bob", _title("Bob"))] + assert report.redirects_created == ["User:Alice", "User:Bob"] + assert report.failed == {} + + +def test_verify_reports_entity_without_username(monkeypatch): + _patch_redirects(monkeypatch) + osw = FakeStoreOsw([{"name": "Alice", "groups": ["user"]}], drop_username=True) + report = _run(osw) + assert "verify" in report.failed + + +def _existing_user(): + return SimpleNamespace( + username="Alice", + uuid=user_uuid(MwUser(name="Alice")), + label=[SimpleNamespace(text="Alice")], + first_name="Alice", + surname="Alice", + orcid=None, + email=set(), + website={"https://example.org"}, + __iris__={ + "organization": ["Item:OSWa"], + "employment_contract_status": "Item:OSWb", + }, + ) + + +def test_prune_run_stores_with_default_and_clears_keys_afterwards(monkeypatch): + _patch_redirects(monkeypatch) + # the fake entity is not a model, so skip the pydantic validation of the param + monkeypatch.setattr( + sync_module, + "OSW", + SimpleNamespace(StoreEntityParam=lambda **kw: SimpleNamespace(**kw)), + ) + cleared = [] + + def fake_clear(osw, title, keys): + cleared.append((title, sorted(keys))) + return True + + monkeypatch.setattr(sync_module, "clear_jsondata_keys", fake_clear) + existing = _existing_user() + osw = FakeStoreOsw( + [{"name": "Alice", "groups": ["user"]}], existing={ITEM_TITLE: existing} + ) + report = _run(osw, prune=True) + assert report.updated == [_title("Alice")] + param = osw.store_params[0] + assert not hasattr(param, "remove_empty") # the store_entity default applies + assert param.entities[0] is existing + assert cleared == [ + (_title("Alice"), ["employment_contract_status", "organization", "website"]) + ] + assert report.pruned == [_title("Alice")] + assert "pruned=1" in report.summary() + + +def test_normal_run_never_clears_keys(monkeypatch): + _patch_redirects(monkeypatch) + monkeypatch.setattr( + sync_module, + "OSW", + SimpleNamespace(StoreEntityParam=lambda **kw: SimpleNamespace(**kw)), + ) + calls = [] + monkeypatch.setattr( + sync_module, "clear_jsondata_keys", lambda *a, **k: calls.append(a) + ) + osw = FakeStoreOsw( + [{"name": "Alice", "groups": ["user"]}], existing={ITEM_TITLE: _existing_user()} + ) + report = _run(osw) + assert calls == [] + assert report.pruned == [] + + +# -------------------------------------------------------------------------- +# merge with the stored page on stage +# -------------------------------------------------------------------------- + + +class OfflineWtPage(WtPage): + """A WtPage that pretends to exist without touching a wiki.""" + + def __init__(self, wtSite: Any = None, title: str = None): + self.wtSite = wtSite + self.title = title + self.exists = True + self._original_content = "" + self.changed: bool = False + self._dict = [] + self._slots: Dict[str, Union[str, dict]] = {"main": ""} + self._slots_changed: Dict[str, bool] = {"main": False} + self._content_model: Dict[str, str] = {"main": "wikitext"} + + +REMOTE_STATUS = "Item:OSWb" + + +def _proposed(user_id, **kw): + args = dict( + uuid=user_id, + username="Alice", + first_name="Alice", + surname="Alice", + label="Alice", + orcid=None, + emails=[], + websites=[], + organizations=[], + ) + args.update(kw) + return ProposedUser(**args) + + +def _json(entity): + return json.loads(entity.json(exclude_none=True)) + + +def test_built_user_has_no_employment_contract_status(): + user = build_user(_proposed(uuid.uuid4(), websites=["https://example.org"])) + assert "employment_contract_status" not in _json(user) + + +def _loaded_user(user_id): + return User( + uuid=user_id, + username="Alice", + first_name="Alice", + surname="Alice", + label=[{"text": "Alice"}], + website={"https://example.org"}, + organization=["Item:OSWa"], + ) + + +def test_normal_update_keeps_remote_employment_contract_status(): + user_id = uuid.uuid4() + remote = { + "type": ["Category:OSWd9aa0bca9b0040d8af6f5c091bf9eec7"], + "uuid": str(user_id), + "username": "Alice", + "first_name": "Alice", + "surname": "Alice", + "label": [{"text": "Alice", "lang": "en"}], + "organization": ["Item:OSWa"], + "employment_contract_status": REMOTE_STATUS, + } + page = OfflineWtPage(title=uuid_to_full_page_title(user_id)) + page.set_slot_content("jsondata", dict(remote)) + entity = _loaded_user(user_id) + apply_update( + entity, + _proposed(user_id, first_name="Alicia", websites=["https://example.org"]), + {"first_name", "websites"}, + ) + assert "employment_contract_status" not in _json(entity) + merged = OSW._apply_overwrite_policy( + OSW._ApplyOverwriteParam( + page=page, entity=entity, policy=True, inplace=False, remove_empty=True + ) + ).get_slot_content("jsondata") + assert merged["employment_contract_status"] == REMOTE_STATUS + assert merged["first_name"] == "Alicia" + + +# -------------------------------------------------------------------------- +# clear_jsondata_keys +# -------------------------------------------------------------------------- + + +class FakePrunePage: + """Stands in for WtPage: holds a jsondata slot and records the writes.""" + + def __init__(self, content): + self.content = content + self.comments = [] + + def get_slot_content(self, slot): + assert slot == "jsondata" + return json.loads(json.dumps(self.content)) + + def set_slot_content(self, slot, content): + assert slot == "jsondata" + self.content = content + + def edit(self, comment=None): + self.comments.append(comment) + + +def _patch_page(monkeypatch, page): + from osw.tools.user_sync import prune + + monkeypatch.setattr(prune, "WtPage", lambda wtSite=None, title=None: page) + + +def test_clear_jsondata_keys_empties_present_keys_only(monkeypatch): + content = { + "website": ["https://example.org"], + "organization": ["Item:OSWa"], + "employment_contract_status": "Item:OSWb", + "short_name": [{"text": "x", "lang": "en"}], + "unknown_key": "x", + } + page = FakePrunePage(content) + _patch_page(monkeypatch, page) + written = clear_jsondata_keys( + SimpleNamespace(site=None), + ITEM_TITLE, + ["website", "organization", "employment_contract_status", "unknown_key"], + ) + assert written is True + assert page.content["website"] == [] + assert page.content["organization"] == [] + assert "employment_contract_status" not in page.content + assert "unknown_key" not in page.content + assert page.content["short_name"] == content["short_name"] + assert page.comments == ["user-sync: prune"] + + +def test_clear_jsondata_keys_leaves_empty_arrays_alone(monkeypatch): + page = FakePrunePage({"website": [], "organization": [], "username": "Alice"}) + _patch_page(monkeypatch, page) + written = clear_jsondata_keys( + SimpleNamespace(site=None), ITEM_TITLE, ["website", "organization"] + ) + assert written is False + assert page.comments == [] + + +def test_clear_jsondata_keys_does_not_add_absent_keys_or_write(monkeypatch): + page = FakePrunePage({"website": [], "username": "Alice"}) + _patch_page(monkeypatch, page) + written = clear_jsondata_keys( + SimpleNamespace(site=None), ITEM_TITLE, ["website", "organization"] + ) + assert written is False + assert page.content == {"website": [], "username": "Alice"} + assert page.comments == [] + + +def test_clear_jsondata_keys_deletes_an_empty_single_valued_key(monkeypatch): + page = FakePrunePage({ + "website": [], + "employment_contract_status": [], + "username": "Alice", + }) + _patch_page(monkeypatch, page) + written = clear_jsondata_keys( + SimpleNamespace(site=None), + ITEM_TITLE, + ["website", "employment_contract_status"], + ) + assert written is True + assert page.content == {"website": [], "username": "Alice"} diff --git a/tests/tools/test_user_sync_store_failures.py b/tests/tools/test_user_sync_store_failures.py new file mode 100644 index 00000000..ad7769e3 --- /dev/null +++ b/tests/tools/test_user_sync_store_failures.py @@ -0,0 +1,271 @@ +"""Failed and partial stores in an accepted sync run (issue 224). + +The model builders, the redirect writer and the plan are replaced with fakes, +so these tests run the orchestrator without a wiki connection. +""" + +import uuid +from types import SimpleNamespace + +import pytest + +from osw.core import OSW +from osw.tools.user_sync import sync +from osw.tools.user_sync.config import SyncConfig +from osw.tools.user_sync.interactive import Prompter, Resolution, ResolvedChange +from osw.tools.user_sync.mapping import ProposedOrganization, ProposedUser +from osw.tools.user_sync.reconcile import REMOVE, FieldDiff, UserChange + + +def _user(name, organizations=()): + return ProposedUser( + uuid=uuid.uuid5(uuid.NAMESPACE_DNS, name), + username=name, + first_name=name, + surname=name, + label=name, + emails=[f"{name}@example.org"], + organizations=list(organizations), + ) + + +def _org(name): + return ProposedOrganization(uuid=uuid.uuid5(uuid.NAMESPACE_URL, name), name=name) + + +def _resolved(user, action): + existing = None if action == "create" else SimpleNamespace(username=user.username) + category = "new" if action == "create" else "gap_fill" + return ResolvedChange(UserChange(user, existing, category), action, set()) + + +def _partial_error(stored, failed): + result = OSW.StoreEntityResult.construct( + change_id="c", + pages={t: object() for t in stored}, + failed={t: RuntimeError(f"boom {t}") for t in failed}, + ) + return OSW.StoreEntityPartialError(result) + + +class FakeOsw: + """Records every store call; ``outcomes`` maps an edit comment to an outcome.""" + + def __init__(self, outcomes=None): + self.outcomes = outcomes or {} + self.stored_payloads = {} + + def store_entity(self, param): + titles = [e.title for e in param.entities] + self.stored_payloads[param.edit_comment] = list(param.entities) + outcome = self.outcomes.get(param.edit_comment) + if callable(outcome): + raise outcome(titles) + if isinstance(outcome, Exception): + raise outcome + + def load_entity(self, titles): + return [SimpleNamespace(username="u") for _ in titles] + + +@pytest.fixture +def run(monkeypatch): + """Run an accepted sync over a fixed plan and return (report, redirects).""" + + def _run(osw, resolved, org_map=None, **config): + redirects = [] + users = [r.change.proposed for r in resolved] + monkeypatch.setattr( + sync, "_build_proposals", lambda c, o, s: (users, org_map or {}, []) + ) + monkeypatch.setattr(sync, "adopt_existing_pages", lambda *a: None) + # The real StoreEntityParam validates entities; the fakes are plain objects. + monkeypatch.setattr( + sync, + "OSW", + SimpleNamespace( + StoreEntityParam=SimpleNamespace, + StoreEntityPartialError=OSW.StoreEntityPartialError, + ), + ) + monkeypatch.setattr(sync, "load_existing_users", lambda o: None) + monkeypatch.setattr(sync, "reconcile", lambda *a, **k: None) + monkeypatch.setattr( + sync, + "resolve_plan", + lambda *a, **k: Resolution(resolved=resolved, accepted=True), + ) + monkeypatch.setattr( + sync, + "build_user", + lambda p: SimpleNamespace( + title=p.full_page_title, organization=list(p.organizations) + ), + ) + monkeypatch.setattr( + sync, + "apply_update", + lambda e, p, f: SimpleNamespace( + title=p.full_page_title, organization=list(p.organizations) + ), + ) + monkeypatch.setattr( + sync, + "build_organization", + lambda o: SimpleNamespace(title=o.full_page_title), + ) + + def fake_redirect(osw_, username, title): + redirects.append(title) + return f"User:{username}" + + monkeypatch.setattr(sync, "ensure_redirect", fake_redirect) + report = sync.run_user_sync( + SyncConfig(domain="d", auto_apply=True, **config), + osw=osw, + prompter=Prompter(lambda p="": "y", lambda t="": None), + ) + return report, redirects + + return _run + + +def test_partial_store_counts_only_stored_titles(run): + ok, bad = _user("ok"), _user("bad") + osw = FakeOsw({ + "user-sync": lambda titles: _partial_error( + stored=[ok.full_page_title], failed=[bad.full_page_title] + ) + }) + report, _ = run(osw, [_resolved(ok, "create"), _resolved(bad, "create")]) + assert report.created == [ok.full_page_title] + assert f"store:{bad.full_page_title}" in report.failed + + +def test_failed_store_counts_nothing_as_created_or_updated(run): + new, old = _user("new"), _user("old") + osw = FakeOsw({"user-sync": RuntimeError("down")}) + report, _ = run(osw, [_resolved(new, "create"), _resolved(old, "update")]) + assert report.created == [] + assert report.updated == [] + assert "store" in report.failed + + +def test_no_redirect_for_a_created_item_that_was_not_stored(run): + ok, bad, old = _user("ok"), _user("bad"), _user("old") + osw = FakeOsw({ + "user-sync": lambda titles: _partial_error( + stored=[ok.full_page_title], + failed=[bad.full_page_title, old.full_page_title], + ) + }) + _, redirects = run( + osw, + [_resolved(ok, "create"), _resolved(bad, "create"), _resolved(old, "update")], + ) + # ``old`` already existed, so its redirect target exists even though the + # update failed. ``bad`` was never stored. + assert sorted(redirects) == sorted([ok.full_page_title, old.full_page_title]) + + +def test_user_is_not_stored_with_a_link_to_a_failed_organization(run): + org = _org("Acme") + linked = _user("linked", organizations=[org.full_page_title]) + osw = FakeOsw({"user-sync: orgs": RuntimeError("down")}) + report, _ = run( + osw, + [_resolved(linked, "create")], + org_map={org.uuid: org}, + link_organizations=True, + ) + assert report.organizations == [] + stored = osw.stored_payloads.get("user-sync", []) + assert "user-sync: orgs" in osw.stored_payloads + assert all(org.full_page_title not in e.organization for e in stored) + # The user itself is still stored, counted and redirected. + assert report.created == [linked.full_page_title] + assert report.redirects_created == ["User:linked"] + + +def test_partial_organization_store_drops_only_the_failed_link(run): + good, gone = _org("Good"), _org("Gone") + linked = _user("linked", organizations=[good.full_page_title, gone.full_page_title]) + osw = FakeOsw({ + "user-sync: orgs": lambda titles: _partial_error( + stored=[good.full_page_title], failed=[gone.full_page_title] + ) + }) + report, _ = run( + osw, + [_resolved(linked, "create")], + org_map={good.uuid: good, gone.uuid: gone}, + link_organizations=True, + ) + assert report.organizations == [good.full_page_title] + assert f"store:{gone.full_page_title}" in report.failed + (stored,) = osw.stored_payloads["user-sync"] + assert stored.organization == [good.full_page_title] + + +def test_accepted_sync_counts_and_redirects_every_user(run): + new, old, same = _user("new"), _user("old"), _user("same") + osw = FakeOsw() + report, redirects = run( + osw, + [_resolved(new, "create"), _resolved(old, "update"), _resolved(same, "skip")], + ) + assert report.accepted is True + assert report.created == [new.full_page_title] + assert report.updated == [old.full_page_title] + assert report.skipped == [same.full_page_title] + assert report.failed == {} + assert sorted(redirects) == sorted([ + new.full_page_title, + old.full_page_title, + same.full_page_title, + ]) + param_entities = osw.stored_payloads["user-sync"] + assert [e.title for e in param_entities] == [ + new.full_page_title, + old.full_page_title, + ] + + +def test_prune_skips_an_update_that_was_not_stored(run, monkeypatch): + ok, bad = _user("ok"), _user("bad") + cleared = [] + monkeypatch.setattr( + sync, "clear_jsondata_keys", lambda o, t, k: cleared.append(t) or True + ) + resolved = [] + for user in (ok, bad): + r = _resolved(user, "update") + r.change.diffs = [FieldDiff("websites", ["x"], [], REMOVE)] + r.apply_fields = {"websites"} + resolved.append(r) + osw = FakeOsw({ + "user-sync": lambda titles: _partial_error( + stored=[ok.full_page_title], failed=[bad.full_page_title] + ) + }) + report, _ = run(osw, resolved, prune=True) + assert cleared == [ok.full_page_title] + assert report.pruned == [ok.full_page_title] + + +def test_failed_organization_keeps_a_link_the_existing_item_has(run): + kept, new = _org("Kept"), _org("New") + user = _user("old", organizations=[kept.full_page_title, new.full_page_title]) + resolved = _resolved(user, "update") + resolved.change.existing = SimpleNamespace( + username="old", __iris__={"organization": [kept.full_page_title]} + ) + osw = FakeOsw({"user-sync: orgs": RuntimeError("down")}) + run( + osw, + [resolved], + org_map={kept.uuid: kept, new.uuid: new}, + link_organizations=True, + ) + (stored,) = osw.stored_payloads["user-sync"] + assert stored.organization == [kept.full_page_title] diff --git a/tests/tools/test_user_sync_sync.py b/tests/tools/test_user_sync_sync.py new file mode 100644 index 00000000..fa1701c9 --- /dev/null +++ b/tests/tools/test_user_sync_sync.py @@ -0,0 +1,189 @@ +"""Unit tests for the orchestrator (dry-run path, no live models).""" + +from types import SimpleNamespace + +from osw.data.import_utility import uuid_to_full_page_title +from osw.tools.user_sync.config import SyncConfig +from osw.tools.user_sync.interactive import Prompter +from osw.tools.user_sync.mapping import user_uuid +from osw.tools.user_sync.sources import MwUser +from osw.tools.user_sync.sync import SyncReport, run_user_sync +from osw.wtsite import WtPage + +ORCID = "0000-0002-6374-9831" + +ORCID_RECORD = { + "person": { + "name": { + "given-names": {"value": "Lukas"}, + "family-name": {"value": "Koschmieder"}, + } + } +} + + +class FakeWtPage(WtPage): + """A WtPage that serves a fixed jsondata slot without a wiki.""" + + def __init__(self, title, jsondata=None): + self.title = title + self._jsondata = jsondata or {} + + def get_slot_content(self, slot, *args, **kwargs): + return self._jsondata + + +class FakeMwSite: + def __init__(self, extra_users=()): + self.extra_users = list(extra_users) + + def api(self, action, **params): + assert action == "query" and params["list"] == "allusers" + return { + "query": { + "allusers": [ + {"name": ORCID, "groups": ["user"]}, + {"name": "SyncBot", "groups": ["user", "bot"]}, + *self.extra_users, + ] + } + } + + +class FakeOsw: + def __init__(self, extra_users=(), existing_pages=None): + self.mw_site = FakeMwSite(extra_users) + self.site = SimpleNamespace( + semantic_search=self._semantic_search, get_page=self._get_page + ) + self.existing_pages = existing_pages or {} + self.loaded = [] + self.store_calls = 0 + + def _semantic_search(self, query): + return [SimpleNamespace(titles=[], truncated=False, next_offset=None, count=0)] + + def _get_missing_page_titles(self, titles, confirm_delay_s=5): + return [t for t in titles if t not in self.existing_pages] + + def _get_page(self, param): + pages = [FakeWtPage(t) for t in param.titles] + return SimpleNamespace(pages=pages, errors=[]) + + def load_entity(self, param): + self.loaded.extend(param.titles) + return [self.existing_pages[t] for t in param.titles] + + def store_entity(self, param): + self.store_calls += 1 + + +class FakeResponse: + status_code = 200 + headers: dict = {} + + def json(self): + return ORCID_RECORD + + def raise_for_status(self): + pass + + +class FakeSession: + def __init__(self): + self.calls = 0 + + def get(self, url, headers=None, timeout=None): + self.calls += 1 + return FakeResponse() + + +class CapturingIO: + def __init__(self): + self.outputs = [] + + def input(self, prompt=""): # pragma: no cover - not used in dry-run + raise AssertionError("dry-run must not prompt") + + def output(self, text=""): + self.outputs.append(text) + + +def test_summary_reads_accepted_state(): + assert SyncReport(accepted=False).summary().startswith("no changes written") + assert SyncReport(accepted=True, created=["x"]).summary().startswith("applied") + + +def test_dry_run_previews_without_writing(): + osw = FakeOsw() + io = CapturingIO() + session = FakeSession() + report = run_user_sync( + SyncConfig(domain="d", dry_run=True), + osw=osw, + prompter=Prompter(io.input, io.output), + session=session, + ) + assert report.accepted is False + assert osw.store_calls == 0 # nothing written + assert session.calls == 1 # ORCID user enriched, bot skipped + joined = "\n".join(io.outputs) + assert ORCID in joined + assert "new=1" in joined # bot excluded, only the ORCID user + assert report.missing_email == [ORCID] # record has no email + assert "no ORCID email" in joined + + +def test_requires_connection(): + try: + run_user_sync(SyncConfig()) + except ValueError as exc: + assert "connection" in str(exc) + else: # pragma: no cover + raise AssertionError("expected ValueError") + + +def _dry_run_output(osw, **config): + io = CapturingIO() + report = run_user_sync( + SyncConfig(domain="d", dry_run=True, **config), + osw=osw, + prompter=Prompter(io.input, io.output), + session=FakeSession(), + ) + return report, "\n".join(io.outputs) + + +def test_preview_lists_skipped_blocked_accounts(): + osw = FakeOsw(extra_users=[{"name": "Mallory", "groups": ["user"], "blockid": 5}]) + _, joined = _dry_run_output(osw) + assert "Skipped blocked accounts (1): Mallory" in joined + assert "new=1" in joined # only the ORCID user + + +def test_include_blocked_keeps_blocked_accounts(): + osw = FakeOsw(extra_users=[{"name": "Mallory", "groups": ["user"], "blockid": 5}]) + _, joined = _dry_run_output(osw, include_blocked=True) + assert "Skipped blocked accounts" not in joined + assert "new=2" in joined + + +def test_existing_page_at_the_deterministic_title_is_not_created_again(): + uid = user_uuid(MwUser(name=ORCID)) + title = uuid_to_full_page_title(uid) + stored = SimpleNamespace( + uuid=uid, + username=None, + label=[], + first_name=None, + surname=None, + orcid=None, + email=set(), + website=set(), + organization=set(), + ) + osw = FakeOsw(existing_pages={title: stored}) + _, joined = _dry_run_output(osw) + assert osw.loaded == [title] + assert "new=0" in joined + assert "gap_fill=1" in joined diff --git a/uv.lock b/uv.lock index e851018e..8580c10f 100644 --- a/uv.lock +++ b/uv.lock @@ -1917,7 +1917,7 @@ wheels = [ [[package]] name = "oold" -version = "0.16.2" +version = "2.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "datamodel-code-generator" }, @@ -1929,9 +1929,9 @@ dependencies = [ { name = "sparqlwrapper" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/44/59/15da5384eab02f8d58cc17be3b820839b0ca76a4dfbdb3225d1cb555f955/oold-0.16.2.tar.gz", hash = "sha256:610192b49c744f5d35f0c9c780019beb2460df55119379e8482d2c484b7bd4e9", size = 702593, upload-time = "2026-05-11T09:44:04.126Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/6d/b6635ad891cde1a11e9ccef3f46c0f8cbe43c683c5e2d99daec6b3d5e587/oold-2.3.0.tar.gz", hash = "sha256:9d3b4de3ad53ebb046b575c772cd009374fedbb964ab3c6fbbfb2f48ce18f1c5", size = 1429319, upload-time = "2026-10-15T09:56:00.491Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/09/7e/ff3368ae3e8efcea19f79ab08c759f6d650c983f8b9b302bfe8d82d3db42/oold-0.16.2-py3-none-any.whl", hash = "sha256:32fd178829fba9a990fd1a1bd34c86378a54d5cdf5f554b10b9fdbdc8bf7715d", size = 189586, upload-time = "2026-05-11T09:44:02.601Z" }, + { url = "https://files.pythonhosted.org/packages/17/fa/91913d09989ad88192d114b47f7743f45c1a18fedfa24097a96917d30040/oold-2.3.0-py3-none-any.whl", hash = "sha256:db9b692142b56471c72b0ee49bd9f814e010f54952fda43d783c9f4fc89ea49f", size = 472728, upload-time = "2026-10-15T09:56:02.23Z" }, ] [[package]] @@ -1948,15 +1948,15 @@ wheels = [ [[package]] name = "opensemantic" -version = "0.2.4" +version = "0.2.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "backports-strenum", marker = "python_full_version < '3.11'" }, { name = "oold" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/09/58/2bdbd07aeb065cbe95d0cdfe024e97e7ca69bfe0d2d49b48ff889466e139/opensemantic-0.2.4.tar.gz", hash = "sha256:1e5f6beac3dc84b04a3de9eb5c05bed9be1b2fd286898df7965be5ced387005e", size = 33204, upload-time = "2026-05-08T12:47:36.822Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d4/87/d55b3d35f3622786de592f2309c70a12c672be147173f0e3e84e37f65dc7/opensemantic-0.2.5.tar.gz", hash = "sha256:07a9c83de7493fc19d092828f27aa3d76c3e965e78f2a28463ee5108842fd2c6", size = 34337, upload-time = "2026-11-13T11:14:41.335Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/99/5c/e39d083d6ab2d036e9ab2c2a9d16ed4653f74402ff8be04efb97fd98ad15/opensemantic-0.2.4-py3-none-any.whl", hash = "sha256:a26c7540751c37010c24c48dfef9a92954301257a2f63c19d864e5aa5010af1c", size = 16596, upload-time = "2026-05-08T12:47:35.699Z" }, + { url = "https://files.pythonhosted.org/packages/a4/08/5685e0552c62f4d3e9373a48066c7d40c1ef21bbfab3c83707b904df3e67/opensemantic-0.2.5-py3-none-any.whl", hash = "sha256:5abf139598c21f53e048d424b03f3200d7a98dcad80077df39e340c599f46b39", size = 16704, upload-time = "2026-11-13T11:14:40.227Z" }, ] [[package]] @@ -1975,15 +1975,16 @@ wheels = [ [[package]] name = "opensemantic-core" -version = "0.57.4.post1000002003001" +version = "0.59.0.post1000004007001" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "oold" }, { name = "opensemantic" }, { name = "pydantic", extra = ["email"] }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e3/a7/d88f31ed639ce75ba58648b5ebcbab795fe82f93341e1d694b8993522789/opensemantic_core-0.57.4.post1000002003001.tar.gz", hash = "sha256:a25b9caee5e267d17569b5d64d7d6183b122e26f2da6ddc898cc224182ff903f", size = 67852, upload-time = "2026-05-04T06:15:27.097Z" } +sdist = { url = "https://files.pythonhosted.org/packages/21/63/1c0a049a28acce2d74cca95117a596f25b563861dd78099f2a081127562e/opensemantic_core-0.59.0.post1000004007001.tar.gz", hash = "sha256:9846fb4763fd25a8d30e24b29645b4c3ba2d52d10d492e6f073d284e1dd45c8d", size = 79184, upload-time = "2026-10-15T12:03:49.052Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/c8/ab45630822479696bd4e7650a7e3a547b782ae3a0b30bfcd04a39e6692d3/opensemantic_core-0.57.4.post1000002003001-py3-none-any.whl", hash = "sha256:6cb35e14e011be95e0ded366d1cc2dd8f547209a68665960ffab530ecb6d7ef4", size = 51538, upload-time = "2026-05-04T06:15:25.887Z" }, + { url = "https://files.pythonhosted.org/packages/c2/19/09bb22ad969a704fe9b2bf4991f24ac243cb380da3c5c9e081e8d5048fd3/opensemantic_core-0.59.0.post1000004007001-py3-none-any.whl", hash = "sha256:8e98fd4dbc2335081c3826a30486d698482fb003f3dc5417a536c32ee25a54c2", size = 59833, upload-time = "2026-10-15T12:03:47.692Z" }, ] [[package]] @@ -2190,11 +2191,11 @@ requires-dist = [ { name = "mwclient", specifier = ">=0.11.0" }, { name = "mwparserfromhell", marker = "extra == 'wikitext'" }, { name = "numpy" }, - { name = "oold", specifier = ">=0.15.0" }, + { name = "oold", specifier = ">=2.3.0" }, { name = "openpyxl", marker = "extra == 'dataimport'" }, - { name = "opensemantic" }, + { name = "opensemantic", specifier = ">=0.2.5" }, { name = "opensemantic-base", specifier = ">=0.42.7" }, - { name = "opensemantic-core", specifier = ">=0.57.4" }, + { name = "opensemantic-core", specifier = ">=0.59.0" }, { name = "osw", extras = ["dataimport"], marker = "extra == 'tutorial'" }, { name = "osw", extras = ["dataimport", "db", "ui", "s3", "wikitext", "mcp", "parallel"], marker = "extra == 'all'" }, { name = "osw", extras = ["wikitext"], marker = "extra == 'mcp'" }, diff --git a/zensical.toml b/zensical.toml index 23f9685e..8a2dfe2b 100644 --- a/zensical.toml +++ b/zensical.toml @@ -18,6 +18,7 @@ nav = [ { "Overview" = "tools/index.md" }, { "CLI" = "tools/cli.md" }, { "MCP" = "tools/mcp.md" }, + { "User Sync (script)" = "tools/user-sync.md" }, { "Configuration" = "tools/configuration.md" }, ]}, { "API Reference" = [