Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
73 commits
Select commit Hold shift + click to select a range
10808a0
fix(core): address follow-up issues in the overwrite policy
LukasGold Sep 3, 2026
cf78de9
fix(core): drop skipped pages from StoreEntityPartialError.stored
LukasGold Sep 3, 2026
217b101
chore(user-sync): scaffold module, example and tracking
raederan Sep 15, 2026
90bf3da
feat(user-sync): enumerate and classify mediawiki users
raederan Sep 16, 2026
5e40525
feat(user-sync): add orcid public api client
raederan Sep 16, 2026
59b5610
feat(user-sync): map sources to user and organization items
raederan Sep 16, 2026
54711e9
feat(user-sync): add reconciliation engine
raederan Sep 16, 2026
e0c54db
feat(user-sync): add interactive preview and resolution
raederan Sep 16, 2026
a2a9d8c
feat(user-sync): store items, redirects, orgs and verify
raederan Sep 16, 2026
f2d011d
feat(user-sync): exclude system accounts and complete idempotent sync
raederan Sep 16, 2026
830d120
docs(user-sync): add user sync guide and finalize example
raederan Sep 16, 2026
31c2f82
feat(user-sync): opt-in extras, standard email, --auto-apply
raederan Sep 17, 2026
4e38b29
feat(user-sync): gate optional fields and opt-in prune removals
raederan Sep 17, 2026
1899d7f
docs(user-sync): document prune and opt-in extras
raederan Sep 17, 2026
828f02c
feat(user-sync): strip employment status and warn on missing email
raederan Sep 17, 2026
87cf85b
fix(user-sync): actually remove pruned fields via empty relations and…
raederan Sep 17, 2026
f09dcf6
fix(core): restore the page cache state once per fetch_schema call
LukasGold Sep 18, 2026
467d11c
fix(package): write packages.json pages in a deterministic order
LukasGold Sep 21, 2026
af682e1
Merge branch 'main' into fix/overwrite-policy-followup
LukasGold Sep 21, 2026
e6acace
test(core): guard the skipped and verified page interaction
LukasGold Sep 21, 2026
90fa310
fix(core): prefer a registered class over a generated one
LukasGold Sep 3, 2026
4ee1e09
fix(core): ignore registered subclasses of the canonical model class
LukasGold Sep 4, 2026
c116199
fix(core): do not warn when a registered subclass was set aside
LukasGold Sep 22, 2026
e39a857
fix(core): use the current class after fetch_schema reloads the module
LukasGold Sep 22, 2026
0ea247c
fix(core): skip non-class attributes when choosing a category class
LukasGold Sep 22, 2026
0017ef0
fix(core): guard the registry conflict check against a non-class
LukasGold Sep 22, 2026
3c9278d
fix(core): skip a page whose category has no usable schema
LukasGold Sep 23, 2026
7e63571
refactor: take the file controllers from opensemantic.core
simontaurus Sep 25, 2026
f3f1080
fix: a wiki file finds its session without a URL
simontaurus Oct 15, 2026
f199283
fix: one session per host and credential, named by host alone
simontaurus Oct 15, 2026
4ecbc7c
fix: path is a field again, in step with local_path
simontaurus Oct 15, 2026
0227107
chore: point at released oold and merged core
simontaurus Oct 15, 2026
c6930fa
chore: require the released core 0.59.0
simontaurus Oct 15, 2026
ae8cf92
chore: update the lock file for oold 2.3.0 and core 0.59.0
simontaurus Oct 15, 2026
01b6531
test: give the test characteristic its parent category
simontaurus Oct 16, 2026
9b76a8f
fix: keep the stored jsonschema when store_entity regenerates it
LukasGold Oct 16, 2026
4d6f440
fix: botfilter for user sync
raederan Nov 3, 2026
9f606f4
fix: select category templates from the schema allOf like the wiki (#…
LukasGold Nov 5, 2026
55e500e
fix: follow properties and items when collecting category templates (…
LukasGold Nov 5, 2026
799bc07
docs: state which schema references the template walk follows (#220)
LukasGold Nov 5, 2026
02570c8
fix(user-sync): drop python-dotenv, use default accounts.pwd.yaml
raederan Nov 6, 2026
1b55f02
ci: run Main on pushes to stage and Integration on PRs into stage
LukasGold Nov 6, 2026
fa07163
Merge pull request #223 from OpenSemanticLab/ci/run-ci-on-stage
LukasGold Nov 6, 2026
8f7cc51
Merge branch 'stage' into feat/file-controllers-from-core
LukasGold Nov 6, 2026
0573271
Merge pull request #214 from OpenSemanticLab/feat/file-controllers-fr…
LukasGold Nov 6, 2026
1a070bf
Merge branch 'stage' into fix/overwrite-policy-followup
LukasGold Nov 6, 2026
1eddbe8
test: drop the FileController type check, it is no longer a model
LukasGold Nov 6, 2026
f524925
Merge pull request #166 from OpenSemanticLab/fix/load-entity-register…
LukasGold Nov 6, 2026
7be59c5
Merge branch 'stage' into fix/overwrite-policy-followup
LukasGold Nov 6, 2026
f247a08
Merge pull request #168 from OpenSemanticLab/fix/overwrite-policy-fol…
LukasGold Nov 6, 2026
04494fa
Merge branch 'stage' into fix/172-package-page-order
LukasGold Nov 6, 2026
5dfdf17
Merge pull request #188 from OpenSemanticLab/fix/172-package-page-order
LukasGold Nov 6, 2026
3ed9616
Merge branch 'stage' into fix/202-category-loop-error-handling
LukasGold Nov 6, 2026
2fc9916
Merge pull request #209 from OpenSemanticLab/fix/202-category-loop-er…
LukasGold Nov 6, 2026
dce6069
Merge pull request #216 from OpenSemanticLab/fix/store-entity-keeps-j…
LukasGold Nov 10, 2026
8671be4
Merge pull request #221 from OpenSemanticLab/fix/category-template-se…
LukasGold Nov 10, 2026
ebfbadb
Merge branch 'stage' into user-item-upd
LukasGold Nov 11, 2026
bd4ffdb
test: set subclass_of for oold 2 in the jsonschema tests
LukasGold Nov 11, 2026
12ecbdd
Merge pull request #225 from OpenSemanticLab/fix/stage-oold-2-subclas…
LukasGold Nov 11, 2026
bc6b0d6
fix(user-sync): align with stage and the tools docs
LukasGold Nov 11, 2026
e97daa4
Merge branch 'stage' into user-item-upd
LukasGold Nov 11, 2026
82ec379
feat: title Property entities by name and refuse a rename on update
LukasGold Nov 6, 2026
0b8dc97
fix(user-sync): count and redirect only stored items (#224)
LukasGold Nov 11, 2026
9b36e4e
fix(user-sync): prune by field type and decide from stored jsondata
LukasGold Nov 12, 2026
ebda514
fix(user-sync): prune a single-valued key stored with an empty value
LukasGold Nov 12, 2026
727032f
Merge remote-tracking branch 'origin/stage' into fix/176-fetch-schema…
LukasGold Nov 12, 2026
477f4ca
Merge pull request #180 from OpenSemanticLab/fix/176-fetch-schema-cache
LukasGold Nov 12, 2026
8a059eb
Merge pull request #174 from OpenSemanticLab/user-item-upd
LukasGold Nov 12, 2026
3464469
build: require opensemantic>=0.2.5 from PyPI
LukasGold Nov 13, 2026
dfc2b4f
Merge branch 'stage' into feat/property-entity-title
LukasGold Nov 13, 2026
723a11b
Merge pull request #226 from OpenSemanticLab/feat/property-entity-title
LukasGold Nov 13, 2026
99584f1
fix(express): connect to http and localhost wikis
LukasGold Nov 18, 2026
244dbcf
Merge pull request #234 from OpenSemanticLab/fix/express-http-scheme
LukasGold Nov 18, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions .github/workflows/integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,17 @@
# INTEGR_TEST_* secrets; where they are unavailable (e.g. forks), the
# credential fixtures make the tests skip instead of fail.
#
# Runs on every pull request into main, so breaking changes surface before the
# release job (gated separately by required review on the pypi environment)
# ever starts. No push trigger: a push to main is already covered by the PR
# that preceded it, so a second run would add nothing.
# Runs on every pull request into main or stage, so breaking changes surface
# before the release job (gated separately by required review on the pypi
# environment) ever starts. stage collects reviewed pull requests before they
# go to main together. No push trigger: a push to main is already covered by
# the PR that preceded it, so a second run would add nothing.

name: Integration

on:
pull_request:
branches: [main]
branches: [main, stage]
workflow_dispatch:

concurrency:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ name: Main

on:
push:
branches: [main]
branches: [main, stage]
pull_request:
types: [opened, synchronize, reopened, ready_for_review]
workflow_dispatch:
Expand Down
4 changes: 2 additions & 2 deletions docs/dev.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,8 @@ excluded by default:

Integration tests run against a live wiki and need credentials; tests
whose credentials are missing skip instead of fail. CI runs them on every
pull request into `main` (workflow `integration.yml`), so they gate the
release rather than run after it.
pull request into `main` or `stage` (workflow `integration.yml`), so they
gate the release rather than run after it.

=== "make"

Expand Down
7 changes: 5 additions & 2 deletions docs/tools/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@

Besides the Python API, osw ships two adapters that talk to a live instance:
the `osw` command line client, and an MCP server for agent clients such as
Claude Code. Both run the same operations from one shared, SDK-free core
(`osw.service`), so a command and its matching tool behave identically. They
Claude Code. This section also covers standalone scripts built on the Python
API. The current one is the [user sync](user-sync.md). The two adapters run
the same operations from one shared, SDK-free core (`osw.service`), so a
command and its matching tool behave identically. They
differ in exactly one way: only the CLI accepts filesystem paths.

## Setup
Expand Down Expand Up @@ -49,4 +51,5 @@ extras are listed in the [Get Started guide](../get-started.md#optional-extras).
| ---- | -------- |
| [CLI](cli.md) | The `.env` quick start, the full command reference, and the global flags |
| [MCP server](mcp.md) | The tool surface, the no-filesystem-access and one-server-per-instance rules, and how to register the server with a client such as Claude Code |
| [User Sync (script)](user-sync.md) | A Python script that creates or updates `User` items from the MediaWiki accounts of an instance, with the options, the data it writes and how it reconciles existing items |
| [Configuration](configuration.md) | What both adapters share: where credentials and settings come from, and the full environment-variable reference |
112 changes: 112 additions & 0 deletions docs/tools/user-sync.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
# User Item Sync

Create or update OSW `User` items from the MediaWiki accounts of an OSL
instance, enriching ORCID users from the public ORCID API. The run is
idempotent and previews every change before writing.

This is a Python script built on the osw library. It is not a command of the
`osw` CLI and not an MCP tool, see [Tools](index.md). It runs from a source
checkout, because `examples/` is not part of the installed package.

## When to use it

OSL instances gain accounts from manual creation and from whitelisted ORCID
single sign-on (whose username is the ORCID iD). Those accounts have no
semantic `User` item until this tool reconciles them.

## Run it

```bash
uv run python examples/user_sync.py --domain your-instance.example.org --dry-run
```

The script reads the credential file given by `--cred-filepath`, otherwise
`accounts.pwd.yaml` in the working directory, otherwise the file named by the
`OSW_CRED_FILEPATH` environment variable. It does not read a `.env` file.
`--domain` falls back to the `OSW_DOMAIN` environment variable. See
[Credentials](configuration.md#credentials) for the file format. Start with
`--dry-run` to preview, then run without it to write after confirming at the
prompt.

To also clean up existing items, add `--prune`. Without it, the sync never
writes `employment_contract_status` and never removes anything. With it,
`employment_contract_status` is removed from the stored jsondata of existing
items. `website` and `organization` are overwritten with an empty list, unless
`--with-websites` or `--with-organizations` enables them. This happens in a
second edit of the page's jsondata after the store. The sync decides from the
stored jsondata, not from the loaded item, which shows the model default for
`employment_contract_status`. A second `--prune` run therefore changes nothing:

```bash
uv run python examples/user_sync.py --domain your-instance.example.org --prune --dry-run
```

Users whose ORCID record exposes no email are listed in a warning at the end of
the run; a missing email never fails the sync.

### Options

By default the tool syncs both ORCID and MediaWiki-native accounts and writes
only core identity plus email.

| Flag | Effect |
| --- | --- |
| `--domain` | Target OSL domain. |
| `--cred-filepath` | Path to `accounts.pwd.yaml`. |
| `--dry-run` | Preview only; never writes. |
| `--auto-apply` | Non-interactive: apply creates, gap-fills and removals; keep existing on conflicts. |
| `--limit N` | Process at most N accounts (testing). |
| `--orcid-only` | Only ORCID-username accounts (excludes `--mw-only`). |
| `--mw-only` | Only non-ORCID (MediaWiki-native) accounts (excludes `--orcid-only`). |
| `--include-system` | Do not skip MediaWiki reserved system accounts. |
| `--no-redirects` | Do not create `User:` redirect pages. |
| `--with-websites` | Also store ORCID researcher URLs (opt-in). |
| `--with-organizations` | Also create and link Organization items from ORCID affiliations (opt-in). |
| `--with-extras` | Enable both websites and organizations. |
| `--include-blocked` | Do not skip blocked MediaWiki accounts. |
| `--prune` | Remove `employment_contract_status` from the stored jsondata of existing items, and overwrite `website` or `organization` with an empty list when their opt-in flag is not given, in a second edit of the page's jsondata after the store (default off; a normal run never writes `employment_contract_status` and never removes anything). |

## What it does

1. Enumerate MediaWiki accounts, skipping blocked accounts (unless
`--include-blocked` is set; the User item of a skipped blocked account is
left unchanged), bots and privileged groups
(`bot`, `sysop`, `bureaucrat`, `interface-admin`) and MediaWiki reserved
system usernames. Accounts whose username is an exact ORCID iD are exempt
from the group filter (so a person's own admin account is synced); other
accounts, such as `<orcid>@bot`, are still excluded by their groups.
2. Enrich ORCID users from `https://pub.orcid.org/v3.0/<id>`: names and email
always; researcher URLs and affiliations only when their flag is enabled.
3. Map each account to a proposed `User` item with a deterministic id
(uuid5 on the ORCID iD, else the username), so re-runs are idempotent.
4. Reconcile against existing items. Existing items are found by `username`
across all `User` items. An item whose deterministic page already exists is
updated and not created again. Each user gets one of four categories: NEW,
GAP_FILL, CONFLICT or UNCHANGED. Each changed field has one of three
statuses: GAP_FILL, CONFLICT or REMOVE. The preview shows a removed field as
`-field`. A user with only gap-fills or removals is GAP_FILL. Gap-fills apply
automatically and conflicts are resolved interactively. Removals appear only
with `--prune`; a normal run never removes anything.
5. Store items, create `User:<username>` redirects to each item, ensure any
linked Organization items, verify by reloading, and warn about users with no
email.

## Data written

| Field | Policy |
| --- | --- |
| `username`, `first_name`, `surname`, `label`, `orcid` | Always (core identity). |
| `email` | Standard: always attempted; missing email is warned, not fatal; never removed. |
| `website` | Opt-in (`--with-websites`); without that flag, on existing items overwritten with an empty list only with `--prune`, in a second jsondata edit after the store. |
| `organization` | Opt-in (`--with-organizations`); without that flag, on existing items overwritten with an empty list only with `--prune`, in a second jsondata edit after the store. |
| `employment_contract_status` | Never written without `--prune`; with `--prune`, removed from the stored jsondata of existing items in a second jsondata edit after the store. |

## Notes

- MediaWiki does not expose other users' email or real name, so ORCID is the
only source of rich data. Non-ORCID accounts get username-based placeholder
names, flagged in the preview. Placeholder names are written only for new
items and are never proposed for an existing item.
- In a source checkout (for developers), the first accepted run autofetches
schemas and may regenerate `src/osw/model/entity.py`. Restore it if you do not
want that change.
30 changes: 30 additions & 0 deletions examples/user_sync.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
"""Create or update OSW User items from MediaWiki accounts and ORCID.

Usage:
uv run python examples/user_sync.py --domain llm4eln.semos.dev --dry-run
"""

from osw.defaults import paths
from osw.express import OswExpress
from osw.tools.user_sync import config_from_args, run_user_sync


def main() -> None:
config = config_from_args()
cred_filepath = config.cred_filepath
if cred_filepath is None and paths.cred_filepath.is_file():
# OswExpress only reads OSW_CRED_FILEPATH; use the default accounts file.
cred_filepath = paths.cred_filepath
osw = OswExpress(domain=config.domain, cred_filepath=cred_filepath)
report = run_user_sync(config, osw=osw)
print(report.summary())
for title in report.created:
print(f" created: {title}")
for title in report.updated:
print(f" updated: {title}")
for key, error in report.failed.items():
print(f" FAILED {key}: {error}")


if __name__ == "__main__":
main()
8 changes: 5 additions & 3 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,11 @@ classifiers = [
"Topic :: Software Development :: Libraries :: Python Modules",
]
dependencies = [
"oold>=0.15.0",
"opensemantic",
"opensemantic.core>=0.57.4",
# registered_credentials(), used to find the wiki of a file without a URL
"oold>=2.3.0",
"opensemantic>=0.2.5",
# the file controllers this module wraps
"opensemantic.core>=0.59.0",
"opensemantic.base>=0.42.7",
"pydantic[email]>=2.12.0",
"datamodel-code-generator==0.51.0",
Expand Down
2 changes: 1 addition & 1 deletion src/osw/cli/ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ def status_instances(ctx: Context) -> dict:
# Not ctx.osw: that one is cached and bound to the active
# instance. These connections are ours alone, so we close each
# one instead of leaving it open for the rest of the command.
connection = ctx.osw_for(config.derive_domain(iri))
connection = ctx.osw_for(config.derive_connect_domain(iri))
entry["connected"] = True
_close_quietly(connection, iri)
except Exception as exc:
Expand Down
10 changes: 10 additions & 0 deletions src/osw/controller/file/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
"""File controllers.

Imported eagerly so the controllers register themselves: a caller that resolves
a file entity through the controller registry then finds the wiki
implementation without importing it by name.
"""

from osw.controller.file import base, local, memory, remote, wiki

__all__ = ["base", "local", "memory", "remote", "wiki"]
69 changes: 6 additions & 63 deletions src/osw/controller/file/base.py
Original file line number Diff line number Diff line change
@@ -1,66 +1,9 @@
from abc import abstractmethod
from typing import IO, Any, Dict
"""File controller interface.

from osw.core import model
The implementation moved to ``opensemantic.core``, which owns the file models.
This module keeps the import path working.
"""

from opensemantic.core.v1 import FileControllerMixin as FileController

class FileController(model.File):
"""Base class for file controllers"""

@abstractmethod
def get(self) -> IO:
"""Reads or downloads a file from an IO object
Make sure to close the IO object after reading

Returns
-------
the IO object of the opened file
"""

@abstractmethod
def put(self, file: IO, **kwargs: Dict[str, Any]):
"""Writes or uploads a file from an IO object

Parameters
-------
file
the IO object to read from
"""

def put_from(self, other: "FileController", **kwargs: Dict[str, Any]):
"""Writes or uploads a file from another file controller
Automatically closes the other file controllers IO object after reading

Parameters
----------
other
another file controller to read from
"""
with other.get() as f:
self.put(f, **kwargs)

def get_to(self, other: "FileController"):
"""Reads or downloads a file to another file controller
Automatically closes this file controllers IO object after reading

Parameters
----------
other
another file controller to write to
"""
with self.get() as f:
other.put(f)

@classmethod
def from_other(cls, other: "FileController", **kwargs: Any) -> "FileController":
"""Create an instance based on another file controller
copying all common attributes

Parameters
----------
other
another file controller to copy from
kwargs
additional attributes to be set
"""
return other.cast(cls, **kwargs)
__all__ = ["FileController"]
Loading
Loading