diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..be1e595 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +.git +node_modules +**/node_modules +**/__pycache__ +pilot/agriculture/.runtime +deployment/.env +deployment/.runtime +deployment/.build +coverage +*.tgz +npm-debug.log* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae102f9..69fff9d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,19 +1,66 @@ -name: CI +name: Registry OpenFn pilot on: pull_request: push: - branches: - - main + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: registry-openfn-pilot-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - test: - runs-on: ubuntu-latest + adaptor-and-service-checks: + runs-on: ubuntu-24.04 + timeout-minutes: 15 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 22 + node-version: 24.19.0 cache: npm - - run: npm ci --ignore-scripts --no-audit --no-fund - - run: npm run check + - name: Install the committed npm dependency graph + run: npm ci --ignore-scripts --no-audit --no-fund + - name: Check Evidence, BREG, event services and compiled pilot jobs + run: npm run check + - name: Check adaptor package contents + run: npm run pack:dry-run + + linux-pilot-acceptance: + needs: adaptor-and-service-checks + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - name: Build pinned images, execute the native worker gate and set up the pilot + # Setup owns the build and suppresses raw upstream subprocess output. + run: ./deployment/pilot.sh setup + - name: Check private mount ownership + run: ./deployment/check-permissions.sh + - name: Run registration, governed correction and verified destination acceptance + run: python3 pilot/smoke.py --timeout 300 + - name: Verify retained records and exact replay after service restart + run: python3 pilot/smoke.py --restart-recovery --timeout 300 + - name: Verify delivery retry, failed-step replay and queued-work recovery + run: python3 pilot/recovery.py --scenario all --timeout 300 + - name: Report service status without logs or credentials + if: failure() + shell: bash + run: | + if [ -f pilot/agriculture/.runtime/prepared.json ]; then + ./deployment/pilot.sh status + fi + - name: Stop application services and retain isolated test data until runner disposal + if: always() + shell: bash + run: | + if [ -f pilot/agriculture/.runtime/prepared.json ]; then + if ! ./deployment/pilot.sh stop >/dev/null 2>&1; then + echo 'Pilot stop did not complete. The isolated CI runner will be disposed.' + fi + fi + # Never upload runtime files, private keys, dataclips or raw service logs. diff --git a/.gitignore b/.gitignore index f730502..51a48ee 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,8 @@ node_modules/ npm-debug.log* .DS_Store coverage/ +pilot/agriculture/.runtime/ +deployment/.env +deployment/.build/ +deployment/.runtime/ +__pycache__/ diff --git a/README.md b/README.md index 38eae21..ad53fff 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,274 @@ -# OpenFn Language Registry Stack +# Registry Stack adaptors for OpenFn -Local OpenFn adaptor monorepo for Registry Stack services. +A self-hosted synthetic pilot connecting upstream OpenFn Lightning to Registry +Stack Base Registry Engine (BREG) and signed Evidence. It demonstrates registration, +reviewed correction, and an idempotent downstream update containing a verified +Boolean. All supplied people, holding names, identifiers and credentials are for +an isolated local pilot. -This repository is shaped for OpenFn local adaptor loading: +The maintained pilot uses: + +- [Registry Evidence adaptor](packages/registry-evidence), using the `evidence` + namespace of the published `@registrystack/client@0.27.0`. +- [Registry BREG adaptor](packages/registry-breg), using the same package's `breg` + namespace for metadata-selected writes, exact lookups and lifecycle actions. +- Upstream Lightning 2.18.2 and websocket worker 1.29.0, packaged with Node + 24.19.0 on glibc and local adaptors. No OpenFn source patch is required. +- Separate BREG and Lightning PostgreSQL databases, Mint, Evidence, an + authenticated event bridge and a small SQLite-backed destination. + +[Image identities](deployment/images.lock.json), the npm lockfile, and bootstrap +[tool checksums](deployment/tools.sha256) pin the delivered dependencies. + +## What the three workflows do + +| Workflow | Authoritative result | Response and boundary | +| --- | --- | --- | +| Registration | Creates one Farm using a stable submission key. | The authenticated synchronous webhook returns its UUID. An exact retry returns the same record; changed content with the same key conflicts. | +| Correction | Creates and submits a name-correction request. | Returns its request UUID. A separate reviewer approves, then explicitly applies it. Submission and approval leave the Farm unchanged. | +| Committed Farm event | Verifies HMAC delivery, requests signed registration Evidence, then updates the destination. | Asynchronous. Only the verified `registered: true`, record reference and revision reach the destination. | + +BREG owns records, permissions, review, revisions, ETags and the transactional +outbox. OpenFn orchestrates these contracts. The Evidence source performs an +exact authorized Farm lookup and returns only `localIdentifier`; Evidence signs +one registration Boolean. The assertion makes no claim about eligibility, +ownership, activity or land rights. An unresolved holding produces unavailable +Evidence, never an invented `false` assertion. + +The bridge validates the signed event before forwarding its minimized input to +OpenFn using a separate committed-event key. Registration and correction clients +receive only the intake key and cannot call that event endpoint. The destination +deduplicates by stable source, event ID and effect and +refuses conflicting replays. Revision checks prevent old deliveries from +regressing current state. Outbox delivery remains at least once. + +## Requirements and first setup + +Use Docker Engine with Compose v2, or a compatible desktop Docker runtime, +Python 3, and a POSIX shell. The build needs access to the pinned upstream images +and Registry Stack release downloads. Host Node is needed for adaptor tests, +but setup performs its Node and Registry CLI work in containers. + +The released Lightning and worker images are `linux/amd64`; Apple Silicon uses +Docker emulation. Native arm64 adaptor loading does not establish arm64 worker +support. See the [container packaging notes](deployment/README.md) for the glibc +worker packaging and architecture limits. Allow enough Docker disk and memory +for two databases and the upstream application builds. + +From this repository: ```sh -export LOCAL_ADAPTORS=true -export OPENFN_ADAPTORS_REPO=/path/to/openfn-language-registry-stack +./deployment/pilot.sh setup +./deployment/pilot.sh status +python3 pilot/smoke.py ``` -With OpenFn kit support from -[OpenFn/kit#1397](https://github.com/OpenFn/kit/pull/1397), this repository can -also be listed before the canonical OpenFn adaptors repo: +Setup builds the pinned images, runs the real native adaptor worker check, +generates private inputs and Evidence fixtures, initializes the isolated BREG +package, applies upstream Lightning migrations, and provisions the pilot +operator and credentials. Webhooks stay disabled until authentication is +attached and checked. A failed step stops setup and identifies the step without +printing raw service output. Setup is not a statement that the full acceptance +journey passed; the smoke must finish with `"status": "passed"`. + +Open [Lightning](http://localhost:4000). The generated login is in +`pilot/agriculture/.runtime/openfn/secrets/operator.json`; open that private file +locally to read the email and password. Do not paste its contents into tickets, +job expressions or logs. There is no shared default password. The same private +folder contains the operator API token, separate intake and committed-event keys, +credential associations and +`project.json`, which records the actual project and trigger UUIDs. + +The smoke creates a fresh synthetic identity once and stores it in owner-only +`pilot/agriculture/.runtime/smoke-state.json`. It exercises registration, +unchanged and conflicting retries, correction submission, separate reviewer +approval/application, event-driven Evidence and destination effects. Repeating +it reuses its original keys and records. It does not clear data or make a new +registration on every run. Direct native HTTP checks are also available in the +[agriculture guide](pilot/agriculture/README.md). + +## Forms and manual review + +The generated project has registration and correction webhook endpoints at +`http://localhost:4000/i/TRIGGER_UUID`. Read their UUIDs from private +`project.json`, and send the intake `webhook-api-key` in `x-api-key`. Keep that key +in the form integration's credential store. Never give form clients the separate +`committed-api-key`; it belongs to the authenticated event bridge. Do not place +either key in the URL. +Registration and correction wait for completion; successful responses include +the final state under `data.data`, while workflow failure uses HTTP 422. The +committed-event trigger acknowledges asynchronously. + +[Synthetic registration input](pilot/agriculture/inputs/registration.json) and +[correction input](pilot/agriculture/inputs/correction.json) show the payloads. +Set the correction's `recordId` to the registered Farm UUID. Keep `submissionId` +stable for the same logical request; use a new one for a deliberately different +request. A correction retry must match all original fields, including its +supporting reference. Workflow outputs return the Farm or request UUID and do +not return the submitted name. + +Approval and application use the separate reviewer credential in the tools +service. OpenFn never receives this private key. Substitute the correction +request UUID, inspect the displayed synthetic proposal, then decide: ```sh -export OPENFN_ADAPTORS_REPO=/path/to/openfn-language-registry-stack,/path/to/adaptors +./deployment/compose.sh run --rm --no-deps tools \ + python3 /opt/pilot/agriculture/review.py inspect REQUEST_UUID +./deployment/compose.sh run --rm --no-deps tools \ + python3 /opt/pilot/agriculture/review.py approve REQUEST_UUID +# The Farm is still unchanged. Inspect the approved request before application. +./deployment/compose.sh run --rm --no-deps tools \ + python3 /opt/pilot/agriculture/review.py inspect REQUEST_UUID +./deployment/compose.sh run --rm --no-deps tools \ + python3 /opt/pilot/agriculture/review.py apply REQUEST_UUID ``` -Lightning and the worker resolve package directories as local adaptors: +The reviewer helper binds the action to its exact inspected proposal version, +effect digest and lifecycle ETag. A stale snapshot is refused. A record ETag is +not a lifecycle ETag. The smoke invokes these explicit steps only for the +synthetic correction it created and verified. + +## Stop, resume and restart recovery -```text -@openfn/language-registry-notary@local -@openfn/language-registry-relay@local +```sh +./deployment/pilot.sh stop +./deployment/pilot.sh start +./deployment/check-permissions.sh +python3 pilot/smoke.py +# After the main smoke passes, exercise service restart with retained records. +python3 pilot/smoke.py --restart-recovery ``` -## Packages +Stop retains named volumes, Registry history and private configuration. Start +checks the retained package and reapplies the supported upstream startup path. +Restart recovery restarts existing application services and verifies the same +records, correction and destination effects. It does not recreate databases or +Mint's shared network namespace. -- [packages/registry-notary](packages/registry-notary): Registry Notary client - helper for OpenFn workflows. -- [packages/registry-relay](packages/registry-relay): Registry Relay protected - read helper for OpenFn workflows. +Always use `deployment/compose.sh` for lower-level commands. It fixes the Compose +project to `registry-openfn-pilot` and preserves the operator UID/GID in ignored +`deployment/.env`. If the pilot moves to another operator, transfer ownership +of its retained private files explicitly and update that file. Do not solve a +permission failure by making keys world-readable. -## Verify +Reset is separate and permanently destroys this synthetic pilot's containers, +volumes, records, history and generated private keys: + +```sh +./deployment/pilot.sh reset --confirm-delete-synthetic-data +``` + +Neither setup, start, stop nor smoke invokes reset. An incomplete preparation or +package activation is retained for inspection. Resolve the exact failed stage +using the owning BREG recovery procedure; do not clear a migration interlock or +remove a partially activated database to make readiness pass. + +## Failure and replay recovery + +| Symptom | Recovery | +| --- | --- | +| Timeout or failed registration/correction job | Inspect the failed work order in Lightning, restore the dependency, then retry the original input with its original submission ID. A timeout can follow a committed mutation. Do not generate a fresh key to hide an unknown outcome. | +| Idempotency or ETag conflict | Confirm whether the payload changed or the baseline is stale. Reuse the original payload for an exact replay. For a deliberate new correction, review current state and create a new submission; do not force a write with a stale ETag. | +| Already-submitted correction | The workflow rereads the request and verifies its original content before reporting `already_submitted`. A missing action alone is not proof of success. | +| Rejected correction | Read the review decision. A repeat of the original submission reports the existing request ID; inspect its current BREG state and decision. It does not resubmit it. Submit a revised proposal with a new submission ID after review, or use a currently advertised revise action through the BREG adaptor. The pilot form does not automatically revise or approve. | +| Authentication failure or expired credentials | Private-key JWT obtains fresh short-lived access tokens through Mint. Check Mint readiness, host clock, client registration and private-key permissions. Correct the OpenFn credential/profile when keys change. Do not paste a temporary access token into job source or rotate unrelated keys during an exact retry. | +| Evidence unavailable, denied or unverifiable | Leave the destination unchanged. Check the authorized exact source lookup, selected requirement, requester credentials and reviewed trust/profile configuration. Unavailable is not a signed negative. Retry the original failed committed-event work order only after the cause is resolved. | +| Failed committed-event workflow after webhook acknowledgement | Use Lightning's failed work order and original input for recovery. BREG's successful HTTP handoff does not imply that the asynchronous workflow succeeded. Destination deduplication makes an exact successful effect replay safe. | +| Pending or dead-letter BREG delivery | Restore bridge/OpenFn availability, inspect delivery status, then explicitly replay the eligible retained dead letter. Preserve event identity and use the listed generation. Do not edit the outbox or manufacture a new event ID. | + +The scoped [recovery helper](pilot/recovery.py) inspects the retained smoke +record's committed-event work orders without printing inputs or credentials. +After repairing the failed dependency, retry its failed step with the same +work order and input: + +```sh +python3 pilot/recovery.py --inspect-smoke +python3 pilot/recovery.py --retry-smoke +python3 pilot/smoke.py +``` + +If the smoke event's workflow succeeded but BREG still retains a dead letter +because its acknowledgement was lost, `python3 pilot/recovery.py --replay-smoke` +replays the eligible generation and verifies the destination effect stays unique. +The helper refuses to replay before the retained workflows have succeeded. + +Only while the dedicated pilot is idle, exercise the failure acceptance scenarios: + +```sh +python3 pilot/recovery.py --scenario all +``` + +This deliberately interrupts selected services and then restores them. It checks +BREG retry/dead-letter replay before OpenFn accepts a delivery, retry of a failed +destination step after acceptance, and retained queued work across application +restarts. Each scenario keeps a separate synthetic identity in private +`.runtime/recovery-state.json`; it removes no records or volumes. Select +`before-accept`, `after-accept`, or `pending-restart` instead of `all` for one case. + +For value-free BREG delivery status and an explicit eligible replay: + +```sh +./deployment/compose.sh run --rm --no-deps tools bregctl webhook list \ + --runtime-config /config/breg/runtime.json +./deployment/compose.sh run --rm --no-deps tools bregctl webhook replay \ + --runtime-config /config/breg/runtime.json \ + --event-id EVENT_UUID --delivery-id DELIVERY_ID --expected-generation GENERATION +``` + +This pilot retains undelivered payloads for one day. BREG removes payloads after +successful delivery; expired payloads cannot be replayed. A BREG operator replay +keeps the event ID but advances delivery generation. Destination effect identity +uses the stable event ID, so that change does not duplicate an applied effect. + +## Private data and diagnostics + +This is a loopback-only, supervised local deployment, not a production internet +configuration. Host ports 4000, 4001 and 4002 bind only to loopback. The tools +service has privileged package, database and reviewer material and is an +operator surface. Retain access controls on the Docker daemon and host files. + +Lightning is configured to retain workflow history and dataclips for seven days. +That includes original webhook inputs and failed-job inputs even when successful +final output is minimized. Configuration is retained only between operations and +removed by the worker at the job output boundary. This removal does not erase +original event data or an input copied into another state property. Avoid logging +record values or copying credentials into `data`. Upstream worker diagnostics +can contain short-lived JWT URLs; lifecycle scripts suppress raw output for this +reason. Do not publish service logs, private runtime folders or support bundles. + +Worker and Lightning external Sentry reporting and usage tracking are disabled. +Registry and Evidence audit volumes, local review snapshots, private smoke state, +and the destination database have their own retention; the seven-day Lightning +setting does not delete them. The destination stores the verified Boolean and +operational identities/revisions, not Farm names or signed assertions. + +## Development and verification + +Use Node 24.19.0 for the same runtime family as the worker: ```sh npm ci --ignore-scripts --no-audit --no-fund npm run check +npm run pack:dry-run ``` + +Tests exercise the published native clients against synthetic HTTP servers, +actual OpenFn compilation/runtime composition, failure redaction, replay and +lifecycle boundaries, bridge authentication and destination persistence. These +checks do not substitute for the container acceptance smoke. The focused +[Linux CI gate](.github/workflows/ci.yml) builds the pinned images, performs setup, +runs the workflow smoke, retained restart recovery and the three bounded failure +scenarios. + +To load the adaptors in another compatible OpenFn installation, configure: + +```sh +export LOCAL_ADAPTORS=true +export OPENFN_ADAPTORS_REPO=/absolute/path/to/openfn-language-registry-stack +``` + +Use `@openfn/language-registry-evidence@local` and +`@openfn/language-registry-breg@local`. The packaged worker also carries local +`common` and `http`; installation is complete at image build time. See the +individual adaptor READMEs for credential schemas, operation options, safe +failure branches and standalone composition rules. diff --git a/deployment/.gitignore b/deployment/.gitignore new file mode 100644 index 0000000..4a0ebcd --- /dev/null +++ b/deployment/.gitignore @@ -0,0 +1,3 @@ +.env +.runtime/ +__pycache__/ diff --git a/deployment/Lightning.Dockerfile b/deployment/Lightning.Dockerfile new file mode 100644 index 0000000..ffc7fc2 --- /dev/null +++ b/deployment/Lightning.Dockerfile @@ -0,0 +1,8 @@ +# syntax=docker/dockerfile:1 +ARG LIGHTNING_IMAGE=openfn/lightning:v2.18.2@sha256:a2327173ece7b5ac4904744acf63f928dd2cbe496788d31266060e85bd359203 +ARG ADAPTORS_IMAGE=registry-openfn-worker:pilot +FROM ${ADAPTORS_IMAGE} AS adaptors +FROM ${LIGHTNING_IMAGE} +COPY --from=adaptors --chown=lightning:root /opt/registry-adaptors /opt/registry-adaptors +ENV OPENFN_ADAPTORS_REPO=/opt/registry-adaptors +ENV LOCAL_ADAPTORS=true diff --git a/deployment/Lightning.Dockerfile.dockerignore b/deployment/Lightning.Dockerfile.dockerignore new file mode 100644 index 0000000..1d085ca --- /dev/null +++ b/deployment/Lightning.Dockerfile.dockerignore @@ -0,0 +1 @@ +** diff --git a/deployment/Postgres.Dockerfile b/deployment/Postgres.Dockerfile new file mode 100644 index 0000000..1ebda62 --- /dev/null +++ b/deployment/Postgres.Dockerfile @@ -0,0 +1,4 @@ +FROM postgres:17.11@sha256:67f41722b7a8cbdb868a44a4995c846eddfdc2973bccb291ce937dce88ad5675 +COPY --chmod=755 deployment/postgres-tls-entrypoint.sh /usr/local/bin/pilot-postgres-entrypoint +ENTRYPOINT ["/usr/local/bin/pilot-postgres-entrypoint"] +CMD ["postgres", "-c", "ssl=on", "-c", "ssl_cert_file=/var/lib/postgresql/tls/server.crt", "-c", "ssl_key_file=/var/lib/postgresql/tls/server.key"] diff --git a/deployment/Postgres.Dockerfile.dockerignore b/deployment/Postgres.Dockerfile.dockerignore new file mode 100644 index 0000000..87e72dd --- /dev/null +++ b/deployment/Postgres.Dockerfile.dockerignore @@ -0,0 +1,3 @@ +** +!deployment +!deployment/postgres-tls-entrypoint.sh diff --git a/deployment/README.md b/deployment/README.md new file mode 100644 index 0000000..26d02d1 --- /dev/null +++ b/deployment/README.md @@ -0,0 +1,98 @@ +# Pilot container packaging + +This directory packages upstream Lightning 2.18.2 and websocket worker 1.29.0 +with the local Evidence and BREG adaptors. Upstream and +Registry Stack image identities are recorded in `images.lock.json`. + +Run `./deployment/build-images.sh` from the repository after its package lock is +current. It builds the worker first and compiles the exact four generated +Lightning job bodies with upstream adaptor export preloading and automatic +imports. The BREG and Evidence jobs must reach their own input validation, and +the destination job executes against a controlled loopback responder. The check +also proves credentials remain available between operations and disappear at +the normal job boundary. +The released engine's real child-process handshake is checked separately under +the image's configured UID and GID, with no network access. +It then builds Lightning, the event services, PostgreSQL TLS setup, and the +checksum-verified Registry Stack 0.27.0 bootstrap tools. +The PostgreSQL check initializes an isolated database from synthetic files owned +by UID1001 with mode 0600, proving that its runtime user can read the staged SQL +without relaxing the operator's file permissions. It mounts no pilot data. +Bootstrap packages come from the signed Debian snapshot dated August 24, 2026, +matching the pinned PostgreSQL base image. + +The worker release is Alpine. The worker Dockerfile copies its compiled +application unchanged into pinned Node 24.19.0 Bookworm, installs the upstream +frozen dependency lock for glibc, and installs the adaptor repository's locked +production dependencies. It does not rebuild or patch OpenFn source. The local +adaptor root contains `registry-evidence`, `registry-breg`, `common`, and `http`. +No dependency installation is performed when the worker starts. + +The released Lightning and worker images provide **linux/amd64 only**. This +pilot therefore pins that platform, including on Apple Silicon under Docker +emulation. The arm64 adaptor layer independently passed both Evidence and BREG native +constructor checks on Node 24.19.0/glibc 2.36. Reproduce with +`docker build --platform linux/arm64 --target adaptors -f deployment/Worker.Dockerfile .`. +An arm64 native-worker claim requires an upstream arm64 image and a +separate real-worker acceptance run. The Node and Registry Stack clients alone +being available for arm64 does not establish that claim. + +`compose.yaml` belongs to the synthetic agriculture pilot. Generate its ignored +runtime configuration using the pilot's setup workflow before using Compose. +It contains persistent independent Lightning and BREG databases. Mint owns a +shared network namespace for BREG, Evidence, worker, bridge and bootstrap tools, +so Registry Stack's explicit development HTTP policy uses loopback. Only the +Lightning UI (4000), BREG event bridge (4001) and BREG reviewer API (4002) are +published, all on host loopback. + +Configuration mounts are scoped to each service. In particular the worker only +receives its Evidence client profile, and Lightning receives its own bootstrap +secrets. The bootstrap tools alone receive all generated Registry Stack inputs. +The wrapper copies the database's private key and initial SQL with postgres +ownership and mode 0600 before invoking the unchanged upstream entrypoint. The +initial SQL applies only when PostgreSQL initializes a new named volume. + +Follow the pilot runbook for migrations, first operator setup, provisioning, +and acceptance. Build checks establish packaging and native execution; they do +not by themselves establish the full workflow or event-delivery journey. + +Use `./deployment/pilot.sh setup` for the complete bootstrap, then `start`, +`stop`, or `status`. Setup preserves an existing completed runtime and verifies +its retained package; it does not clear a failed preparation or reset records. +A failed step stops startup, and raw runtime diagnostics are suppressed because +upstream errors can contain short-lived worker-token URLs. Database migrations +use the upstream release API before starting Lightning. Workflows are provisioned +disabled, webhook authentication is attached, then authentication is checked +before enabling triggers. Readiness waits are bounded. + +`./deployment/compose.sh` is the supported Compose entrypoint. It creates ignored +`deployment/.env` containing the current operator UID and GID and uses that same +identity for OpenFn, Registry Stack, bootstrap and event-service containers. +The worker image also bakes a matching passwd/group entry when needed because +upstream starts its engine children with an empty environment. Rebuild images +with `build-images.sh` when transferring the pilot to a different operator. +Generated private files remain owned by that operator with restrictive modes. +The separate PostgreSQL entrypoint still drops to the image's postgres user. +`volume-init` adjusts ownership only inside the pilot's named audit, destination +and worker-cache volumes. It preserves metadata when ownership and permissions +already match, so resumed starts do not invalidate active audit writers. +No other host paths or volumes are touched. + +Run `./deployment/check-permissions.sh` after preparing or transferring a pilot. +It checks each service's private bind mount under the selected UID without +printing contents. Moving a pilot to another operator requires an explicit +ownership transfer of retained private files and updating `deployment/.env`; +the wrapper refuses to silently use another operator's settings. Do not make +private keys world-readable. Local image checks use a non-default UID/GID on +macOS/OrbStack. The focused GitHub Actions gate runs fresh setup, the complete +workflow journey, retained restart and failure recovery on Ubuntu 24.04. + +External Sentry reporting is explicitly disabled for worker and Lightning, and +Lightning usage tracking is disabled. The worker uses its dedicated writable +cache rather than writing to the packaged application. + +Reset is deliberately separate and destructive. Only +`./deployment/pilot.sh reset --confirm-delete-synthetic-data` removes the fixed +`registry-openfn-pilot` project volumes and its marked synthetic `.runtime` +directory. No reset is performed by setup, start or stop. Run the normal pilot +smoke journey after setup to establish full workflow behavior. diff --git a/deployment/Services.Dockerfile b/deployment/Services.Dockerfile new file mode 100644 index 0000000..c593767 --- /dev/null +++ b/deployment/Services.Dockerfile @@ -0,0 +1,7 @@ +# syntax=docker/dockerfile:1 +FROM node:24.19.0-bookworm@sha256:4196d66a565c6f195728d9952f161f4adfe2ad753052a08b7ec7f1c5a6bda42b +WORKDIR /app +COPY services /app/services +RUN mkdir /data && chown node:node /data +USER node +CMD ["node", "services/event-bridge/src/server.js"] diff --git a/deployment/Services.Dockerfile.dockerignore b/deployment/Services.Dockerfile.dockerignore new file mode 100644 index 0000000..063f68c --- /dev/null +++ b/deployment/Services.Dockerfile.dockerignore @@ -0,0 +1,5 @@ +** +!services +!services/** +services/**/node_modules +services/**/.env diff --git a/deployment/Tools.Dockerfile b/deployment/Tools.Dockerfile new file mode 100644 index 0000000..744f6eb --- /dev/null +++ b/deployment/Tools.Dockerfile @@ -0,0 +1,25 @@ +# Debian 13 matches the released Linux tools' runtime. PostgreSQL client is +# already supplied by the same pinned image used for the pilot database. +FROM postgres:17.11@sha256:67f41722b7a8cbdb868a44a4995c846eddfdc2973bccb291ce937dce88ad5675 +# Match the immutable Debian snapshot recorded by the pinned base image. +# Retain signature verification; historical repository metadata has expired. +RUN sed -i \ + -e 's|http://deb.debian.org/debian-security|http://snapshot.debian.org/archive/debian-security/20260824T000000Z|' \ + -e 's|http://deb.debian.org/debian|http://snapshot.debian.org/archive/debian/20260824T000000Z|' \ + /etc/apt/sources.list.d/debian.sources && \ + rm /etc/apt/sources.list.d/pgdg.list && \ + apt-get -o Acquire::Check-Valid-Until=false -o APT::Update::Error-Mode=any update && \ + apt-get install -y --no-install-recommends python3 python3-yaml curl ca-certificates openssl && \ + rm -rf /var/lib/apt/lists/* +COPY deployment/tools.sha256 /tmp/tools.sha256 +RUN set -eu; cd /tmp; \ + for tool in breg bregctl evidence evidencectl mint; do \ + curl --fail --location --retry 3 --output "$tool-v0.27.0-linux-amd64" "https://github.com/registrystack/registry-stack/releases/download/v0.27.0/$tool-v0.27.0-linux-amd64"; \ + done; \ + sha256sum --check tools.sha256; \ + for tool in breg bregctl evidence evidencectl mint; do \ + install -m 0755 "$tool-v0.27.0-linux-amd64" "/usr/local/bin/$tool"; \ + rm "$tool-v0.27.0-linux-amd64"; \ + done +ENTRYPOINT [] +CMD ["python3", "--version"] diff --git a/deployment/Tools.Dockerfile.dockerignore b/deployment/Tools.Dockerfile.dockerignore new file mode 100644 index 0000000..ee4501b --- /dev/null +++ b/deployment/Tools.Dockerfile.dockerignore @@ -0,0 +1,3 @@ +** +!deployment +!deployment/tools.sha256 diff --git a/deployment/Worker.Dockerfile b/deployment/Worker.Dockerfile new file mode 100644 index 0000000..62bbb91 --- /dev/null +++ b/deployment/Worker.Dockerfile @@ -0,0 +1,49 @@ +# syntax=docker/dockerfile:1 +ARG WORKER_IMAGE=openfn/ws-worker:v1.29.0@sha256:827f4897d120c322d977da7bc2810174332eb5ba4902fdb7279178812d787c18 +ARG NODE_IMAGE=node:24.19.0-bookworm@sha256:4196d66a565c6f195728d9952f161f4adfe2ad753052a08b7ec7f1c5a6bda42b +FROM ${WORKER_IMAGE} AS upstream +FROM ${NODE_IMAGE} AS worker-runtime +# Upstream publishes Alpine only. Keep its compiled application unchanged and +# re-install the exact locked dependencies for glibc. Never rebuild its source. +COPY --from=upstream /app /app +WORKDIR /app +RUN corepack enable && CI=true pnpm install --frozen-lockfile --force --ignore-scripts +WORKDIR /app/packages/ws-worker +USER node +CMD ["node", "./dist/start.js"] + +FROM ${NODE_IMAGE} AS adaptors +WORKDIR /opt/registry-adaptors +COPY package.json package-lock.json ./ +COPY packages/registry-evidence/package.json packages/registry-evidence/package.json +COPY packages/registry-breg/package.json packages/registry-breg/package.json +RUN npm ci --omit=dev --ignore-scripts --no-audit --no-fund +COPY deployment/check-native.cjs /opt/registry-adaptors/check-native.cjs +RUN node /opt/registry-adaptors/check-native.cjs +COPY packages/registry-evidence/src packages/registry-evidence/src +COPY packages/registry-evidence/configuration-schema.json packages/registry-evidence/configuration-schema.json +COPY packages/registry-breg/src packages/registry-breg/src +COPY packages/registry-breg/configuration-schema.json packages/registry-breg/configuration-schema.json +# Local loading expects packages/, including common and HTTP. +RUN ln -s ../node_modules/@openfn/language-common packages/common && \ + ln -s ../node_modules/@openfn/language-http packages/http + +FROM worker-runtime AS worker +ARG PILOT_UID=1000 +ARG PILOT_GID=1000 +USER root +COPY --from=adaptors --chown=node:node /opt/registry-adaptors /opt/registry-adaptors +COPY deployment/check-worker.mjs /opt/registry-adaptors/check-worker.mjs +COPY deployment/check-engine.mjs /opt/registry-adaptors/check-engine.mjs +COPY pilot/lightning/project.mjs /opt/registry-adaptors/pilot-project.mjs +ENV OPENFN_ADAPTORS_REPO=/opt/registry-adaptors +ENV LOCAL_ADAPTORS=true +RUN node --experimental-vm-modules /opt/registry-adaptors/check-worker.mjs +# The upstream engine starts children with an empty environment. Their Node +# runtime needs a passwd entry for the mapped operator UID during startup. +RUN if ! getent group "$PILOT_GID" >/dev/null; then groupadd --gid "$PILOT_GID" pilot; fi && \ + if ! getent passwd "$PILOT_UID" >/dev/null; then \ + useradd --uid "$PILOT_UID" --gid "$PILOT_GID" --home-dir /var/cache/openfn --no-create-home --shell /usr/sbin/nologin pilot; \ + fi && \ + mkdir -p /var/cache/openfn && chown "$PILOT_UID:$PILOT_GID" /var/cache/openfn && chmod 0700 /var/cache/openfn +USER ${PILOT_UID}:${PILOT_GID} diff --git a/deployment/Worker.Dockerfile.dockerignore b/deployment/Worker.Dockerfile.dockerignore new file mode 100644 index 0000000..51710b1 --- /dev/null +++ b/deployment/Worker.Dockerfile.dockerignore @@ -0,0 +1,21 @@ +** +!package.json +!package-lock.json +!packages +!packages/registry-evidence +!packages/registry-evidence/package.json +!packages/registry-evidence/configuration-schema.json +!packages/registry-evidence/src +!packages/registry-evidence/src/** +!packages/registry-breg +!packages/registry-breg/package.json +!packages/registry-breg/configuration-schema.json +!packages/registry-breg/src +!packages/registry-breg/src/** +!deployment +!deployment/check-worker.mjs +!deployment/check-engine.mjs +!deployment/check-native.cjs +!pilot +!pilot/lightning +!pilot/lightning/project.mjs diff --git a/deployment/build-images.sh b/deployment/build-images.sh new file mode 100755 index 0000000..65debc3 --- /dev/null +++ b/deployment/build-images.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker build --platform linux/amd64 --target worker --build-arg "PILOT_UID=$(id -u)" --build-arg "PILOT_GID=$(id -g)" -f deployment/Worker.Dockerfile -t registry-openfn-worker:pilot . +docker run --rm --platform linux/amd64 --entrypoint node registry-openfn-worker:pilot --experimental-vm-modules /opt/registry-adaptors/check-worker.mjs +docker run --rm --platform linux/amd64 --network none --entrypoint node registry-openfn-worker:pilot /opt/registry-adaptors/check-engine.mjs +docker build --platform linux/amd64 -f deployment/Lightning.Dockerfile -t registry-openfn-lightning:pilot . +docker build --platform linux/amd64 -f deployment/Services.Dockerfile -t registry-openfn-services:pilot . +docker build --platform linux/amd64 -f deployment/Postgres.Dockerfile -t registry-openfn-postgres:pilot . +sh deployment/check-postgres.sh +docker build --platform linux/amd64 -f deployment/Tools.Dockerfile -t registry-openfn-tools:pilot . diff --git a/deployment/check-engine.mjs b/deployment/check-engine.mjs new file mode 100644 index 0000000..d7b14ac --- /dev/null +++ b/deployment/check-engine.mjs @@ -0,0 +1,24 @@ +// Exercise the released engine's child-process handshake, including its empty +// child environment, under the image's configured runtime identity. +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import createEngine from '/app/packages/engine-multi/dist/index.js'; + +const repoDir = await mkdtemp(join(tmpdir(), 'pilot-engine-gate-')); +const timeout = setTimeout(() => { + console.error('Released worker engine did not initialize within 20 seconds.'); + process.exit(1); +}, 20_000); +let engine; +try { + engine = await createEngine({ + repoDir, memoryLimitMb: 500, maxWorkers: 2, + workerValidationTimeout: 5000, workerValidationRetries: 1, + }); + console.log(JSON.stringify({ verified: 'released-worker-engine-startup', uid: process.getuid(), gid: process.getgid() })); +} finally { + if (engine) await engine.destroy(); + clearTimeout(timeout); + await rm(repoDir, { recursive: true, force: true }); +} diff --git a/deployment/check-native.cjs b/deployment/check-native.cjs new file mode 100644 index 0000000..75e23ee --- /dev/null +++ b/deployment/check-native.cjs @@ -0,0 +1,11 @@ +const assert = require('node:assert/strict'); +const { generateKeyPairSync, createHash } = require('node:crypto'); +const { createRequire } = require('node:module'); +const { evidence, breg } = createRequire('/opt/registry-adaptors/package.json')('@registrystack/client'); +const { publicKey } = generateKeyPairSync('ec', { namedCurve: 'prime256v1' }); +const jwk = publicKey.export({ format: 'jwk' }); +const kid = createHash('sha256').update(JSON.stringify({ crv: jwk.crv, kty: jwk.kty, x: jwk.x, y: jwk.y })).digest('base64url'); +new evidence.EvidenceClient({ baseUrl: 'http://127.0.0.1:8080', trustedJwks: { keys: [{ ...jwk, kid, alg: 'ES256' }] }, revokedKeyIds: [], token: { static: 'synthetic-test-value' } }); +new breg.BaseRegistryClient({ baseUrl: 'http://127.0.0.1:8090' }); +assert.ok(process.report.getReport().header.glibcVersionRuntime); +console.log(JSON.stringify({ verified: 'native-evidence-and-breg-constructors', architecture: process.arch, node: process.version, glibc: process.report.getReport().header.glibcVersionRuntime })); diff --git a/deployment/check-permissions.sh b/deployment/check-permissions.sh new file mode 100755 index 0000000..1b876cc --- /dev/null +++ b/deployment/check-permissions.sh @@ -0,0 +1,27 @@ +#!/bin/sh +# Check bind readability as the runtime UIDs without exposing file contents. +set -eu +cd "$(dirname "$0")/.." +runtime_root="$(pwd)/pilot/agriculture/.runtime" +node_image='node:24.19.0-bookworm@sha256:4196d66a565c6f195728d9952f161f4adfe2ad753052a08b7ec7f1c5a6bda42b' +check_tree() { + service_name="$1" + runtime_uid="$2" + subtree="$3" + if ! docker run --rm --platform linux/amd64 --user "$runtime_uid" \ + --mount "type=bind,src=$runtime_root/$subtree,dst=/check,readonly" \ + "$node_image" node -e 'const fs=require("fs"); function visit(p){fs.accessSync(p,fs.constants.R_OK); const s=fs.statSync(p); if(s.isDirectory()){fs.accessSync(p,fs.constants.X_OK); for(const n of fs.readdirSync(p))visit(p+"/"+n);}} try{visit("/check")}catch{process.exit(1)}'; then + printf '%s\n' "Permission check failed for $service_name (UID $runtime_uid, $subtree)." >&2 + printf '%s\n' 'Run setup as the operator who owns these generated files and use deployment/compose.sh to match container UID/GID. If transferring an existing pilot, explicitly transfer private file ownership. Do not make secrets public. See deployment/README.md.' >&2 + exit 1 + fi +} +runtime_uid="$(id -u):$(id -g)" +check_tree mint "$runtime_uid" mint +check_tree breg "$runtime_uid" breg +check_tree evidence "$runtime_uid" evidence +check_tree worker "$runtime_uid" evidence-client +check_tree bridge "$runtime_uid" bridge +check_tree destination "$runtime_uid" destination +check_tree lightning "$runtime_uid" openfn/secrets +printf '%s\n' 'All service-specific secret bind mounts are readable by their runtime UIDs.' diff --git a/deployment/check-postgres.sh b/deployment/check-postgres.sh new file mode 100644 index 0000000..baeba94 --- /dev/null +++ b/deployment/check-postgres.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# Exercise the Linux host-operator to postgres UID handoff using only synthetic +# inputs inside a disposable, network-disabled container. Never mount pilot data. +set -eu +docker run --rm -i --platform linux/amd64 --network none --user root \ + --tmpfs /var/lib/postgresql/data:rw,nosuid,nodev,size=256m \ + --env POSTGRES_HOST_AUTH_METHOD=trust --entrypoint sh \ + registry-openfn-postgres:pilot -se <<'CHECK' +install -d -o 1001 -g 1001 -m 0700 /config/postgres +openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj /CN=synthetic-check \ + -keyout /config/postgres/server.key -out /config/postgres/server.crt >/dev/null 2>&1 +printf '%s\n' 'CREATE ROLE pilot_permission_probe NOLOGIN;' >/config/postgres/bootstrap.sql +chown 1001:1001 /config/postgres/* +chmod 0600 /config/postgres/* +# Reproduce the old direct bind's ownership before the wrapper stages its copy. +install -o 1001 -g 1001 -m 0600 /config/postgres/bootstrap.sql /docker-entrypoint-initdb.d/bootstrap.sql +if gosu postgres test -r /docker-entrypoint-initdb.d/bootstrap.sql; then + echo 'PostgreSQL ownership check did not establish its negative precondition.' >&2 + exit 1 +fi +/usr/local/bin/pilot-postgres-entrypoint postgres -c listen_addresses='' >/tmp/postgres-check.log 2>&1 & +database_pid=$! +trap 'kill "$database_pid" 2>/dev/null || true; wait "$database_pid" 2>/dev/null || true' EXIT +attempt=0 +while [ "$attempt" -lt 45 ]; do + if ! kill -0 "$database_pid" 2>/dev/null; then + echo 'PostgreSQL initialization failed under its runtime UID; raw diagnostics withheld.' >&2 + exit 1 + fi + if [ "$(gosu postgres psql -U postgres -d postgres -Atqc "SELECT count(*) FROM pg_roles WHERE rolname = 'pilot_permission_probe'" 2>/dev/null || true)" = 1 ]; then + test "$(stat -c '%u:%g:%a' /docker-entrypoint-initdb.d/bootstrap.sql)" = "$(id -u postgres):$(id -g postgres):600" + test "$(stat -c '%u:%g:%a' /config/postgres/bootstrap.sql)" = '1001:1001:600' + gosu postgres test -r /var/lib/postgresql/tls/server.key + echo 'PostgreSQL initialized with private operator-owned inputs and a private runtime-owned copy.' + exit 0 + fi + attempt=$((attempt + 1)) + sleep 1 +done +echo 'PostgreSQL ownership check timed out.' >&2 +exit 1 +CHECK diff --git a/deployment/check-worker.mjs b/deployment/check-worker.mjs new file mode 100644 index 0000000..153f394 --- /dev/null +++ b/deployment/check-worker.mjs @@ -0,0 +1,105 @@ +// Executes the generated Lightning job bodies with the released worker's exact +// compiler autoinjection and runtime linker. No live Registry Stack is needed. +import assert from 'node:assert/strict'; +import { createRequire } from 'node:module'; +import { readFileSync } from 'node:fs'; +import http from 'node:http'; +import { once } from 'node:events'; +import { projectDocument } from './pilot-project.mjs'; + +const require = createRequire('/opt/registry-adaptors/package.json'); +const client = require('@registrystack/client'); +assert.equal(typeof client.evidence.EvidenceClient, 'function'); +new client.breg.BaseRegistryClient({ baseUrl: 'http://127.0.0.1:8090' }); +assert.equal(JSON.parse(readFileSync('/opt/registry-adaptors/node_modules/@registrystack/client/package.json')).version, '0.27.0'); +assert.ok(process.report.getReport().header.glibcVersionRuntime, 'worker must use glibc'); +const { default: compile, preloadAdaptorExports } = await import('/app/packages/compiler/dist/index.js'); +const { default: run } = await import('/app/packages/runtime/dist/index.js'); +const paths = { + '@openfn/language-registry-breg': '/opt/registry-adaptors/packages/registry-breg', + '@openfn/language-registry-evidence': '/opt/registry-adaptors/packages/registry-evidence', + '@openfn/language-http': '/opt/registry-adaptors/packages/http', +}; +const linker = Object.fromEntries(Object.entries(paths).map(([name, path]) => [name, { path }])); + +async function executeJob(job, input) { + const name = job.adaptor.slice(0, job.adaptor.lastIndexOf('@')); + // Matches engine-multi/src/worker/thread/compile.ts in ws-worker v1.29.0. + // No explicit common import is added: each chosen adaptor must export the + // actual helpers the Lightning job uses, including fn and execute. + const exports = await preloadAdaptorExports(paths[name]); + const { code } = compile(job.body, { + name: job.name, + 'add-imports': { adaptors: [{ name, exports, exportAll: true }] }, + }); + assert.ok(code.includes(name), `compiler did not inject ${name}`); + return run({ + workflow: { steps: [{ id: job.id, name: job.name, expression: code, linker }], start: job.id }, + options: { start: job.id }, + }, input, { + strict: false, + linker: { repo: '/tmp/pilot-compile-gate', modules: linker, cacheKey: `pilot-${job.id}-${process.pid}` }, + }); +} + +const document = projectDocument({ + accessProfile: 'openfn-service', + createFarmOperation: 'records.farm.create', + createCorrectionOperation: 'records.name-correction.create', + requirement: 'urn:example:requirement:holding-registered:v1', + registeredConcept: 'urn:example:concept:holding-registered:registered', +}); +assert.equal(document.workflows.length, 3); +const jobs = document.workflows.flatMap(workflow => workflow.jobs); +assert.equal(jobs.length, 4); +for (const job of jobs.filter(job => !job.adaptor.startsWith('@openfn/language-http'))) { + const result = await executeJob(job, { data: {}, configuration: { fixture: 'synthetic-test-value' } }); + // These exact bodies validate before any registry call. Reaching their own + // validation proves the full injected module graph linked and evaluated. + const expected = job.name === 'verify-registration' + ? /A committed registry event with identity, revision, and selector is required/ + : /A stable submissionId is required/; + assert.match(JSON.stringify(result.errors), expected, `generated job ${job.name} did not reach its input validation`); + assert.equal('configuration' in result, false); +} + +// Execute the exact HTTP destination body against a controlled loopback server. +// This tests its injected post/fn exports and its request options, without any +// external call or pilot destination side effect. +let received = 0; +const server = http.createServer(async (req, res) => { + const chunks = []; + for await (const chunk of req) chunks.push(chunk); + const body = JSON.parse(Buffer.concat(chunks)); + assert.equal(req.url, '/updates'); + assert.equal(req.method, 'POST'); + assert.equal(req.headers['x-destination-key'], 'synthetic-destination-key'); + assert.deepEqual(body, { revision: 1, values: { registered: true } }); + received += 1; + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ status: 'accepted', revision: 1 })); +}); +server.listen(0, '127.0.0.1'); +await once(server, 'listening'); +try { + const result = await executeJob(jobs.find(job => job.name === 'record-update'), { + data: { revision: 1, values: { registered: true } }, + configuration: { baseUrl: `http://127.0.0.1:${server.address().port}`, apiKey: 'synthetic-destination-key' }, + }); + assert.equal(result.errors, undefined, JSON.stringify(result.errors)); + assert.equal(received, 1); + assert.deepEqual(result.data, { status: 'accepted', revision: 1 }); + assert.equal('configuration' in result, false); +} finally { + server.closeAllConnections(); + await new Promise(resolve => server.close(resolve)); +} + +const composed = await executeJob({ + id: 'composed-operation-boundary', name: 'composed-operation-boundary', adaptor: '@openfn/language-registry-breg@local', + body: `fn(state => ({...state, data: {first: !!state.configuration.fixture}}));\nfn(state => ({...state, data: {...state.data, second: !!state.configuration.fixture}}));`, +}, { data: {}, configuration: { fixture: 'synthetic-test-value' } }); +assert.equal(composed.errors, undefined, JSON.stringify(composed.errors)); +assert.deepEqual(composed.data, { first: true, second: true }); +assert.equal('configuration' in composed, false); +console.log(JSON.stringify({ verified: 'generated-lightning-jobs-and-native-adaptors', workflows: 3, jobBodies: 4, node: process.version, architecture: process.arch, glibc: process.report.getReport().header.glibcVersionRuntime })); diff --git a/deployment/compose.sh b/deployment/compose.sh new file mode 100755 index 0000000..b71b427 --- /dev/null +++ b/deployment/compose.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# A stable project name prevents commands from selecting another Compose stack. +set -eu +base_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +if [ ! -f "$base_dir/.env" ]; then + (umask 077; printf 'PILOT_UID=%s\nPILOT_GID=%s\n' "$(id -u)" "$(id -g)" > "$base_dir/.env") +fi +configured_uid=$(sed -n 's/^PILOT_UID=\([0-9][0-9]*\)$/\1/p' "$base_dir/.env") +configured_gid=$(sed -n 's/^PILOT_GID=\([0-9][0-9]*\)$/\1/p' "$base_dir/.env") +if [ "$configured_uid" != "$(id -u)" ] || [ "$configured_gid" != "$(id -g)" ]; then + printf '%s\n' 'deployment/.env must contain this operator UID and GID. Review ownership before moving an existing runtime to another operator; do not widen secret permissions.' >&2 + exit 1 +fi +exec docker compose --project-name registry-openfn-pilot --env-file "$base_dir/.env" --file "$base_dir/compose.yaml" "$@" diff --git a/deployment/compose.yaml b/deployment/compose.yaml new file mode 100644 index 0000000..c332440 --- /dev/null +++ b/deployment/compose.yaml @@ -0,0 +1,209 @@ +name: registry-openfn-pilot + +services: + volume-init: + image: node:24.19.0-bookworm@sha256:4196d66a565c6f195728d9952f161f4adfe2ad753052a08b7ec7f1c5a6bda42b + platform: linux/amd64 + user: root + environment: + PILOT_UID: ${PILOT_UID:-1000} + PILOT_GID: ${PILOT_GID:-1000} + # Do not touch metadata of already-owned audit files on resumed starts. + command: [sh, -ec, 'find /mint-audit /evidence-audit /destination-data /worker-cache \( ! -uid "$$PILOT_UID" -o ! -gid "$$PILOT_GID" \) -exec chown "$$PILOT_UID:$$PILOT_GID" {} +; find /mint-audit /evidence-audit /destination-data /worker-cache -maxdepth 0 ! -perm 0700 -exec chmod 0700 {} +'] + volumes: + - mint-audit:/mint-audit + - evidence-audit:/evidence-audit + - destination-data:/destination-data + - worker-cache:/worker-cache + + lightning-db: + image: postgres:17.11@sha256:67f41722b7a8cbdb868a44a4995c846eddfdc2973bccb291ce937dce88ad5675 + env_file: ../pilot/agriculture/.runtime/openfn/postgres.env + volumes: + - lightning-db:/var/lib/postgresql/data + healthcheck: + test: [CMD-SHELL, 'pg_isready -U "$$POSTGRES_USER" -d "$$POSTGRES_DB"'] + interval: 3s + timeout: 3s + retries: 30 + restart: unless-stopped + + breg-db: + image: registry-openfn-postgres:pilot + platform: linux/amd64 + env_file: ../pilot/agriculture/.runtime/breg-db/postgres.env + volumes: + - breg-db:/var/lib/postgresql/data + - ../pilot/agriculture/.runtime/breg-db:/config/postgres:ro + healthcheck: + test: [CMD-SHELL, 'pg_isready -U "$$POSTGRES_USER" -d "$$POSTGRES_DB"'] + interval: 3s + timeout: 3s + retries: 30 + restart: unless-stopped + + lightning: + user: "${PILOT_UID:-1000}:${PILOT_GID:-1000}" + image: registry-openfn-lightning:pilot + platform: linux/amd64 + env_file: ../pilot/agriculture/.runtime/openfn/lightning.env + environment: + PORT: '4000' + URL_SCHEME: http + URL_HOST: localhost + URL_PORT: '4000' + USAGE_TRACKING_ENABLED: 'false' + LOCAL_ADAPTORS: 'true' + OPENFN_ADAPTORS_REPO: /opt/registry-adaptors + volumes: + - ../pilot/lightning:/opt/pilot/lightning:ro + - ../pilot/agriculture/.runtime/openfn/secrets:/run/pilot-secrets + ports: ['127.0.0.1:4000:4000'] + depends_on: + lightning-db: {condition: service_healthy} + healthcheck: + test: [CMD, curl, --fail, --silent, http://127.0.0.1:4000/health_check] + interval: 5s + timeout: 5s + retries: 60 + restart: unless-stopped + + # Mint can start from offline configuration before the BREG package is built. + # Owning this namespace keeps development HTTP on loopback without weakening + # Registry Stack's destination or token endpoint policies. + mint: + user: "${PILOT_UID:-1000}:${PILOT_GID:-1000}" + image: ghcr.io/registrystack/mint:v0.27.0@sha256:cf16981eb2e992722c0e00f3c2b68c85a875843c838bd7ba7f728b456322282e + platform: linux/amd64 + command: [serve, --config, /config/mint/mint.json] + volumes: + - ../pilot/agriculture/.runtime/mint:/config/mint:ro + - mint-audit:/var/lib/mint + depends_on: + volume-init: {condition: service_completed_successfully} + ports: + - '127.0.0.1:4001:8081' + - '127.0.0.1:4002:8090' + restart: unless-stopped + + breg: + user: "${PILOT_UID:-1000}:${PILOT_GID:-1000}" + image: ghcr.io/registrystack/breg:v0.27.0@sha256:222e4ee3e27f7ee5f237dc06693a07773222b39ef15051abb27bac2cae632586 + platform: linux/amd64 + network_mode: service:mint + command: [--config, /config/breg/runtime.json] + environment: + SSL_CERT_FILE: /config/breg/secrets/database-ca.pem + volumes: + - ../pilot/agriculture/.runtime/breg:/config/breg:ro + depends_on: + mint: {condition: service_started} + breg-db: {condition: service_healthy} + restart: unless-stopped + + evidence: + user: "${PILOT_UID:-1000}:${PILOT_GID:-1000}" + image: ghcr.io/registrystack/evidence:v0.27.0@sha256:9b9ddae7fe9d1185f9423a7ca57ea612d178a154df5f42d26f91ff62dc03f4d8 + platform: linux/amd64 + network_mode: service:mint + command: [serve, --runtime, /config/evidence/runtime.yaml] + volumes: + - ../pilot/agriculture/.runtime/evidence:/config/evidence:ro + - evidence-audit:/var/lib/evidence + depends_on: + mint: {condition: service_started} + breg: {condition: service_started} + restart: unless-stopped + + worker: + user: "${PILOT_UID:-1000}:${PILOT_GID:-1000}" + image: registry-openfn-worker:pilot + platform: linux/amd64 + network_mode: service:mint + env_file: ../pilot/agriculture/.runtime/openfn/worker.env + environment: + OPENFN_ADAPTORS_REPO: /opt/registry-adaptors + WORKER_LIGHTNING_SERVICE_URL: ws://lightning:4000/worker + WORKER_STATE_PROPS_TO_REMOVE: configuration + WORKER_REPO_DIR: /var/cache/openfn/repo + WORKER_SENTRY_DSN: '' + SENTRY_DSN: '' + volumes: + - ../pilot/agriculture/.runtime/evidence-client:/config/evidence-client:ro + - worker-cache:/var/cache/openfn + depends_on: + mint: {condition: service_started} + lightning: {condition: service_healthy} + restart: unless-stopped + + bridge: + user: "${PILOT_UID:-1000}:${PILOT_GID:-1000}" + image: registry-openfn-services:pilot + platform: linux/amd64 + network_mode: service:mint + command: [node, services/event-bridge/src/server.js] + env_file: ../pilot/agriculture/.runtime/openfn/bridge.env + environment: + PORT: '8081' + ALLOW_HTTP: 'true' + BREG_HMAC_KEY_FILE: /config/bridge/breg-hmac-key + BREG_EXPECTED_EVENTS_FILE: /config/bridge/expected-events.json + BREG_EXPECTED_ENTITY: farm + BREG_ALLOWED_VALUE_FIELDS_FILE: /config/bridge/allowed-value-fields.json + OPENFN_API_KEY_FILE: /config/bridge/openfn-api-key + MAX_BODY_BYTES: '65536' + MAX_DELIVERY_SKEW_SECONDS: '300' + OPENFN_TIMEOUT_MS: '3000' + volumes: + - ../pilot/agriculture/.runtime/bridge:/config/bridge:ro + depends_on: + mint: {condition: service_started} + lightning: {condition: service_healthy} + restart: unless-stopped + + destination: + user: "${PILOT_UID:-1000}:${PILOT_GID:-1000}" + image: registry-openfn-services:pilot + platform: linux/amd64 + command: [node, services/destination/src/server.js] + environment: + PORT: '8082' + DESTINATION_API_KEY_FILE: /config/destination/api-key + DESTINATION_DB_PATH: /data/destination.sqlite + volumes: + - ../pilot/agriculture/.runtime/destination:/config/destination:ro + - destination-data:/data + depends_on: + volume-init: {condition: service_completed_successfully} + restart: unless-stopped + + tools: + user: "${PILOT_UID:-1000}:${PILOT_GID:-1000}" + image: registry-openfn-tools:pilot + platform: linux/amd64 + network_mode: service:mint + profiles: [tools] + working_dir: /workspace + environment: + SSL_CERT_FILE: /config/breg/secrets/database-ca.pem + volumes: + - ..:/workspace:ro + - ../pilot/agriculture:/opt/pilot/agriculture:ro + - ../pilot/agriculture/.runtime:/workspace/pilot/agriculture/.runtime + - ../pilot/agriculture/.runtime/breg:/config/breg + - ../pilot/agriculture/.runtime/evidence:/config/evidence + - ../pilot/agriculture/.runtime/mint:/config/mint + - ../pilot/agriculture/.runtime/evidence-client:/config/evidence-client + depends_on: + mint: {condition: service_started} + breg-db: {condition: service_healthy} + entrypoint: [] + command: [python3, --version] + +volumes: + lightning-db: + breg-db: + mint-audit: + evidence-audit: + worker-cache: + destination-data: diff --git a/deployment/images.lock.json b/deployment/images.lock.json new file mode 100644 index 0000000..5b516fc --- /dev/null +++ b/deployment/images.lock.json @@ -0,0 +1,15 @@ +{ + "schemaVersion": 1, + "resolvedAt": "2026-09-09", + "platform": "linux/amd64", + "lightning": "openfn/lightning:v2.18.2@sha256:a2327173ece7b5ac4904744acf63f928dd2cbe496788d31266060e85bd359203", + "worker": "openfn/ws-worker:v1.29.0@sha256:827f4897d120c322d977da7bc2810174332eb5ba4902fdb7279178812d787c18", + "node": "node:24.19.0-bookworm@sha256:4196d66a565c6f195728d9952f161f4adfe2ad753052a08b7ec7f1c5a6bda42b", + "postgres": "postgres:17.11@sha256:67f41722b7a8cbdb868a44a4995c846eddfdc2973bccb291ce937dce88ad5675", + "registryClientVersion": "0.27.0", + "toolsDebianSnapshot": "20260824T000000Z", + "notes": "Released Lightning and worker images publish linux/amd64 only. The worker application is copied unchanged to the pinned glibc Node image; native optional dependencies are installed from the released worker lockfile. No arm64 worker claim is made.", + "breg": "ghcr.io/registrystack/breg:v0.27.0@sha256:222e4ee3e27f7ee5f237dc06693a07773222b39ef15051abb27bac2cae632586", + "evidence": "ghcr.io/registrystack/evidence:v0.27.0@sha256:9b9ddae7fe9d1185f9423a7ca57ea612d178a154df5f42d26f91ff62dc03f4d8", + "mint": "ghcr.io/registrystack/mint:v0.27.0@sha256:cf16981eb2e992722c0e00f3c2b68c85a875843c838bd7ba7f728b456322282e" +} diff --git a/deployment/pilot.sh b/deployment/pilot.sh new file mode 100755 index 0000000..3ab7478 --- /dev/null +++ b/deployment/pilot.sh @@ -0,0 +1,123 @@ +#!/bin/sh +# Isolated synthetic pilot lifecycle. It never selects another Compose project. +set -eu +base_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +repo_dir=$(CDPATH= cd -- "$base_dir/.." && pwd) +runtime_dir="$repo_dir/pilot/agriculture/.runtime" +node_image='node:24.19.0-bookworm@sha256:4196d66a565c6f195728d9952f161f4adfe2ad753052a08b7ec7f1c5a6bda42b' +operator_uid="$(id -u):$(id -g)" +cd "$repo_dir" + +compose() { "$base_dir/compose.sh" "$@"; } +step() { + label="$1" + shift + printf '%s\n' "$label" + # Upstream startup failures can include worker JWT URLs. Do not emit or retain + # raw subprocess output from lifecycle actions; each failed step is identified. + if ! "$@" >/dev/null 2>&1; then + printf '%s\n' "$label failed. Raw runtime output was suppressed to protect credentials." >&2 + exit 1 + fi +} +node_command() { + command_name="$1" + docker run --rm --platform linux/amd64 --user "$operator_uid" \ + --network registry-openfn-pilot_default \ + --env OPENFN_URL=http://lightning:4000 \ + --mount "type=bind,src=$repo_dir,dst=/workspace,readonly" \ + --mount "type=bind,src=$runtime_dir,dst=/workspace/pilot/agriculture/.runtime" \ + --workdir /workspace "$node_image" \ + node pilot/lightning/provision.mjs "$command_name" +} +wait_for() { + compose run --rm --no-deps tools python3 /workspace/deployment/wait-http.py "$1" "$2" "${3:-90}" +} +require_prepared() { + if [ ! -f "$runtime_dir/prepared.json" ]; then + printf '%s\n' 'No completed pilot configuration. Run deployment/pilot.sh setup first.' >&2 + exit 1 + fi +} +start_services() { + require_prepared + step 'Validate Compose configuration.' compose config --quiet + step 'Prepare persistent pilot volume ownership.' compose up --no-deps --abort-on-container-exit --exit-code-from volume-init volume-init + step 'Start isolated databases and Mint.' compose up -d --wait --wait-timeout 90 lightning-db breg-db mint + step 'Wait for Mint.' wait_for http://127.0.0.1:8091/ready Mint + step 'Apply upstream Lightning database migrations.' compose run --rm --no-deps lightning /app/bin/lightning eval 'Lightning.Release.migrate()' + step 'Verify or initialize the exact BREG package.' compose run --rm --no-deps tools python3 /opt/pilot/agriculture/initialize.py + step 'Start BREG, Evidence and Lightning.' compose up -d breg evidence lightning + step 'Wait for BREG.' wait_for http://127.0.0.1:8090/ready BREG + step 'Wait for Evidence.' wait_for http://127.0.0.1:8080/ready Evidence + step 'Wait for Lightning.' wait_for http://lightning:4000/health_check Lightning 120 + step 'Prepare the pilot operator through upstream contexts.' compose exec -T lightning /app/bin/lightning rpc 'Code.eval_file("/opt/pilot/lightning/admin.exs"); :ok' + step 'Provision disabled workflows and credentials through the API.' node_command provision + step 'Attach webhook authentication through upstream contexts.' compose exec -T lightning /app/bin/lightning rpc 'Code.eval_file("/opt/pilot/lightning/admin.exs"); :ok' + step 'Verify webhook protection before enabling triggers.' node_command enable + step 'Start worker, verified event bridge and destination.' compose up -d worker bridge destination + step 'Wait for the worker.' wait_for http://127.0.0.1:2222/livez Worker + step 'Wait for the event bridge.' wait_for http://127.0.0.1:8081/healthz Bridge + step 'Wait for the destination.' wait_for http://destination:8082/healthz Destination + printf '%s\n' 'Pilot ready at http://localhost:4000. Run the pilot smoke journey to verify its full behavior.' +} + +case "${1:-}" in + setup) + step 'Build pinned images and run native worker gates.' "$base_dir/build-images.sh" + if [ ! -e "$runtime_dir" ]; then + staging_dir="$base_dir/.runtime/preparation" + if [ -e "$staging_dir" ]; then + printf '%s\n' 'A previous preparation staging directory exists. Inspect it before retrying; it was preserved.' >&2 + exit 1 + fi + (umask 077; mkdir -p "$base_dir/.runtime"; mkdir "$staging_dir") + step 'Generate private synthetic runtime inputs and run Evidence fixtures offline.' \ + docker run --rm --platform linux/amd64 --network none --user "$operator_uid" \ + --mount "type=bind,src=$repo_dir,dst=/workspace,readonly" \ + --mount "type=bind,src=$staging_dir,dst=/output" \ + registry-openfn-tools:pilot python3 /workspace/pilot/agriculture/prepare.py \ + --bin-dir /usr/local/bin --output /output/runtime + mv "$staging_dir/runtime" "$runtime_dir" + rmdir "$staging_dir" + fi + require_prepared + # Prepare only private local files, before any Docker network is needed. + step 'Prepare private OpenFn credentials.' \ + docker run --rm --platform linux/amd64 --network none --user "$operator_uid" \ + --mount "type=bind,src=$repo_dir,dst=/workspace,readonly" \ + --mount "type=bind,src=$runtime_dir,dst=/workspace/pilot/agriculture/.runtime" \ + --workdir /workspace "$node_image" node pilot/lightning/provision.mjs prepare + step 'Check runtime secret ownership.' "$base_dir/check-permissions.sh" + start_services + ;; + start) start_services ;; + stop) + require_prepared + compose stop + printf '%s\n' 'Pilot stopped. Databases, history and private configuration are retained.' + ;; + status) + require_prepared + compose ps + ;; + reset) + if [ "${2:-}" != '--confirm-delete-synthetic-data' ] || [ "$#" -ne 2 ]; then + printf '%s\n' 'Reset permanently deletes this pilot databases, history and private keys. To authorize it, use: deployment/pilot.sh reset --confirm-delete-synthetic-data' >&2 + exit 2 + fi + require_prepared + # Validate the marker before deleting only this fixed pilot directory. + docker run --rm --platform linux/amd64 --network none --user "$operator_uid" \ + --mount "type=bind,src=$runtime_dir,dst=/runtime,readonly" "$node_image" \ + node -e 'const p=require("/runtime/prepared.json");if(p.schema!=="synthetic-agriculture-pilot/v1"||p.version!=="0.27.0")process.exit(1)' + step 'Delete explicitly authorized synthetic pilot containers and volumes.' compose down --volumes + # The path is fixed above, not supplied by an argument or an environment var. + rm -rf -- "$runtime_dir" + printf '%s\n' 'Synthetic pilot state deleted. Run setup for a new isolated pilot.' + ;; + *) + printf '%s\n' 'Usage: deployment/pilot.sh setup|start|stop|status|reset --confirm-delete-synthetic-data' >&2 + exit 2 + ;; +esac diff --git a/deployment/postgres-tls-entrypoint.sh b/deployment/postgres-tls-entrypoint.sh new file mode 100644 index 0000000..6911ce1 --- /dev/null +++ b/deployment/postgres-tls-entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +# Generated credentials remain owner-only on the host. Prepare postgres-owned +# copies inside this container before the upstream entrypoint drops root. +install -d -o postgres -g postgres -m 0700 /var/lib/postgresql/tls +install -o postgres -g postgres -m 0600 /config/postgres/server.key /var/lib/postgresql/tls/server.key +install -o postgres -g postgres -m 0644 /config/postgres/server.crt /var/lib/postgresql/tls/server.crt +install -o postgres -g postgres -m 0600 /config/postgres/bootstrap.sql /docker-entrypoint-initdb.d/bootstrap.sql +exec /usr/local/bin/docker-entrypoint.sh "$@" diff --git a/deployment/tools.sha256 b/deployment/tools.sha256 new file mode 100644 index 0000000..b0eb338 --- /dev/null +++ b/deployment/tools.sha256 @@ -0,0 +1,5 @@ +d6fc64014079206da5f849dd9775375e7f302564890c10d232565c2dfb47c060 breg-v0.27.0-linux-amd64 +3a73c1e0bfab13c25ec6ceee7a39c59d0f5c3ecf561576a7b4fe246d1ba99679 bregctl-v0.27.0-linux-amd64 +524df3e7de28032045a5fcace079697c8f1d597093b755195c14d4b420614096 evidence-v0.27.0-linux-amd64 +0f9c0886c2c418159505afbdc69d185858c6b3c2e4ee4d762ea03da9fffd871b evidencectl-v0.27.0-linux-amd64 +d50591bae3760126ae1c34e6d77311d475161d30d591646506a0d14f3f09d9d9 mint-v0.27.0-linux-amd64 diff --git a/deployment/wait-http.py b/deployment/wait-http.py new file mode 100644 index 0000000..89fc0ca --- /dev/null +++ b/deployment/wait-http.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +"""Bounded readiness checks. Never print response bodies or credential headers.""" +import sys +import time +import urllib.error +import urllib.request + +url = sys.argv[1] +label = sys.argv[2] +timeout = int(sys.argv[3]) if len(sys.argv) > 3 else 90 +deadline = time.monotonic() + timeout +while time.monotonic() < deadline: + try: + with urllib.request.urlopen(url, timeout=3) as response: + if response.status == 200: + print(f'{label} ready.') + sys.exit(0) + except (urllib.error.URLError, TimeoutError, OSError): + pass + time.sleep(1) +print(f'{label} did not become ready within {timeout} seconds.', file=sys.stderr) +sys.exit(1) diff --git a/package-lock.json b/package-lock.json index f742857..2f0f839 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,8 +11,11 @@ "workspaces": [ "packages/*" ], + "dependencies": { + "@openfn/language-http": "7.3.3" + }, "engines": { - "node": ">=20.18.1" + "node": ">=22.12.0" } }, "node_modules/@babel/runtime": { @@ -232,14 +235,14 @@ } }, "node_modules/@openfn/compiler": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/@openfn/compiler/-/compiler-1.2.5.tgz", - "integrity": "sha512-X0JcfC0rPfD/RJwMGr3uNOp+YiixJ86V1JintarndCUNOUI+Not9wq6sLQSz51YCMXbMezMhbY+QZN0e7WyzNA==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@openfn/compiler/-/compiler-1.3.0.tgz", + "integrity": "sha512-Zg0R7RE179cS31/4CtnOlmHglyi0Loj5gHPdCGHOfACOMhOL3UM6v65kQZ/dCJyFW3bcoh8T7Z7vnnwIQ/6nZw==", "dev": true, "license": "ISC", "dependencies": { "@openfn/describe-package": "0.1.6", - "@openfn/lexicon": "^2.0.0", + "@openfn/lexicon": "^2.4.0", "@openfn/logger": "1.1.2", "acorn": "^8.16.0", "ast-types": "^0.14.2", @@ -268,9 +271,9 @@ } }, "node_modules/@openfn/language-common": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/@openfn/language-common/-/language-common-3.3.3.tgz", - "integrity": "sha512-i5uaM0inXROpU25C2zXV6h4rzEq19oMSGmg939vu3JBD5FxopyWR6ZMXAEho5EiFv7QPIHGueTdQbCQg+V5fWg==", + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@openfn/language-common/-/language-common-3.3.4.tgz", + "integrity": "sha512-I5kKmrjr30BKoL0RzuLGKcw7IpDLxpQtfBS6ljhH4MBE4TJ9BUTGXFU46QI7RbkpTNdsi6mtwuRnl2tioM9Dpw==", "license": "LGPL-3.0-or-later", "dependencies": { "ajv": "^8.20.0", @@ -280,9 +283,44 @@ "http-status-codes": "^2.3.0", "jsonpath-plus": "^10.3.0", "lodash": "^4.18.1", - "undici": "^7.24.7" + "undici": "^7.28.0" + } + }, + "node_modules/@openfn/language-http": { + "version": "7.3.3", + "resolved": "https://registry.npmjs.org/@openfn/language-http/-/language-http-7.3.3.tgz", + "integrity": "sha512-WlNuFIJNbGYMvf9Nw6iF4advlgAb2hLUpuqDNP1xdosINyKHDVA9/MlPN8hTECcCVenE7eJTYdEM3two4BB14w==", + "license": "LGPLv3", + "dependencies": { + "@openfn/language-common": "3.3.5", + "cheerio": "1.0.0-rc.12", + "cheerio-tableparser": "^1.0.1" + } + }, + "node_modules/@openfn/language-http/node_modules/@openfn/language-common": { + "version": "3.3.5", + "resolved": "https://registry.npmjs.org/@openfn/language-common/-/language-common-3.3.5.tgz", + "integrity": "sha512-v3fx5RX+sCt03gMxxtAEki6xgGf6z4ipp7wZMOHkhp/OMB4xBveD8g/LRgYjzLAYtRAMk7rmCcwNL7dGPv5/zg==", + "license": "LGPL-3.0-or-later", + "dependencies": { + "ajv": "^8.20.0", + "csv-parse": "^5.4.0", + "csvtojson": "^2.0.13", + "date-fns": "^2.25.0", + "http-status-codes": "^2.3.0", + "jsonpath-plus": "^10.3.0", + "lodash": "^4.18.1", + "undici": "^7.29.0" } }, + "node_modules/@openfn/language-registry-breg": { + "resolved": "packages/registry-breg", + "link": true + }, + "node_modules/@openfn/language-registry-evidence": { + "resolved": "packages/registry-evidence", + "link": true + }, "node_modules/@openfn/language-registry-notary": { "resolved": "packages/registry-notary", "link": true @@ -292,9 +330,9 @@ "link": true }, "node_modules/@openfn/lexicon": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/@openfn/lexicon/-/lexicon-2.2.1.tgz", - "integrity": "sha512-bskftXnRE0GMrzD1+zPor9YegKeKjohQRBy4f5upedxLLjjdWCtHUrn85jV8/3W3dYuqJr3pC8tdDOUfIgsR2w==", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@openfn/lexicon/-/lexicon-2.4.0.tgz", + "integrity": "sha512-a46A0UHvHBMeszh5nluvU8I4cAaT2nZRZ6gydl2INasW8CPQ5KRHhZx9ZmzgkuA8M2f3hvcmiLyD4j+iMdbGAw==", "dev": true, "license": "ISC", "dependencies": { @@ -319,9 +357,9 @@ } }, "node_modules/@openfn/runtime": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/@openfn/runtime/-/runtime-1.9.3.tgz", - "integrity": "sha512-m4HeHYdWONWHjfayhKne7CgTIGx6l+p7MANevVIbrZs4dmFJb+g+DcHcikfRvW7S9sA2xBFnXK7FyMl/QjD7AQ==", + "version": "1.9.5", + "resolved": "https://registry.npmjs.org/@openfn/runtime/-/runtime-1.9.5.tgz", + "integrity": "sha512-y8cFF4Z0uKoKxwQJNwk15EaAYlfIqgvc0FU9ADHT/sZ3nT90qM4Y0+XGkd3g4OuO0ri1JOKkjj8Dc5VTX4uCVg==", "dev": true, "license": "ISC", "dependencies": { @@ -332,6 +370,68 @@ "source-map": "^0.7.6" } }, + "node_modules/@registrystack/client": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@registrystack/client/-/client-0.27.0.tgz", + "integrity": "sha512-wELi6zYe/r6VzunfgLGWUwAVPhLhtF1fPTArCuHxwdjlBfZhCMYbBh/QWv20M97NJm6hgp7f0y4mYqolXl/tvA==", + "license": "Apache-2.0", + "engines": { + "node": ">=22.12.0" + }, + "optionalDependencies": { + "@registrystack/client-darwin-arm64": "0.27.0", + "@registrystack/client-linux-arm64-gnu": "0.27.0", + "@registrystack/client-linux-x64-gnu": "0.27.0" + } + }, + "node_modules/@registrystack/client-darwin-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@registrystack/client-darwin-arm64/-/client-darwin-arm64-0.27.0.tgz", + "integrity": "sha512-utqr47FzIIzAcIHq26aRgKfPdJ0zAhuDifE3iMh24BeyrD9erLQQpufFZvOLrvkNZ6ffr4RPSz5XvmhwbL4icQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/@registrystack/client-linux-arm64-gnu": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@registrystack/client-linux-arm64-gnu/-/client-linux-arm64-gnu-0.27.0.tgz", + "integrity": "sha512-TNAVMleJhX9IwZExoq0OrUw/HgrRCC7ehvIgtz6fVDzOLbgDDBAf/pSujNeLipeqL74BYmy0SHRoQJ8h28Hwdw==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/@registrystack/client-linux-x64-gnu": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@registrystack/client-linux-x64-gnu/-/client-linux-x64-gnu-0.27.0.tgz", + "integrity": "sha512-80UiF58s5ydBSR1XZQNfxd5ybKQj9LlvocfzRu06qGNkyjEsV/pjhZzMktQhQl1ji83bdtPPZbmvNvej9vWZwA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=22.12.0" + } + }, "node_modules/@types/mute-stream": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/@types/mute-stream/-/mute-stream-0.0.1.tgz", @@ -373,9 +473,9 @@ } }, "node_modules/acorn": { - "version": "8.17.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.17.0.tgz", - "integrity": "sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz", + "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", "dev": true, "license": "MIT", "bin": { @@ -655,6 +755,12 @@ "dev": true, "license": "MIT" }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "license": "ISC" + }, "node_modules/braces": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", @@ -707,6 +813,50 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/cheerio": { + "version": "1.0.0-rc.12", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz", + "integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==", + "license": "MIT", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "htmlparser2": "^8.0.1", + "parse5": "^7.0.0", + "parse5-htmlparser2-tree-adapter": "^7.0.0" + }, + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/cheerio-tableparser": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/cheerio-tableparser/-/cheerio-tableparser-1.0.1.tgz", + "integrity": "sha512-SCSWdMoFvIue0jdFZqRNPXDCZ67vuirJEG3pfh3AAU2hwxe/qh1EQUkUNPWlZhd6DMjRlTfcpcPWbaowjwRnNQ==", + "license": "MIT" + }, "node_modules/chokidar": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", @@ -989,6 +1139,34 @@ "node-fetch": "^2.7.0" } }, + "node_modules/css-select": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz", + "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", + "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/csv-parse": { "version": "5.6.0", "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.6.0.tgz", @@ -1083,6 +1261,61 @@ "node": ">=8" } }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -1110,6 +1343,18 @@ "dev": true, "license": "MIT" }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -1335,6 +1580,25 @@ "node": ">=8" } }, + "node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, "node_modules/http-status-codes": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/http-status-codes/-/http-status-codes-2.3.0.tgz", @@ -1501,9 +1765,9 @@ } }, "node_modules/js-yaml": { - "version": "3.14.2", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", - "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", + "version": "3.15.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.15.1.tgz", + "integrity": "sha512-S99WuO3HlhO3XN41EtYUNl9zzXjoJx7QvmipxsJVxtCBT0YHEFy+iOJhjSvrmV12nYhWpZaM8lPHkJm0yUMbag==", "dev": true, "license": "MIT", "dependencies": { @@ -1746,6 +2010,18 @@ "node": ">=0.10.0" } }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, "node_modules/p-defer": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz", @@ -1846,6 +2122,43 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz", + "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==", + "license": "MIT", + "dependencies": { + "domhandler": "^5.0.3", + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5/node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/path-exists": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-5.0.0.tgz", @@ -2406,9 +2719,9 @@ } }, "node_modules/undici": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz", - "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==", + "version": "7.29.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.1.tgz", + "integrity": "sha512-RYONW2MeafgYlkVOKYKkA/Ag7BmXqgIWCa8t1m0JcxrQg9pI9lEqRhAOruOBCbAohOa/gkCF+iPi9hrgvTzu6Q==", "license": "MIT", "engines": { "node": ">=20.18.1" @@ -2547,9 +2860,9 @@ } }, "node_modules/yargs": { - "version": "17.7.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", - "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "version": "17.7.3", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.3.tgz", + "integrity": "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==", "dev": true, "license": "MIT", "dependencies": { @@ -2620,6 +2933,38 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "packages/registry-breg": { + "name": "@openfn/language-registry-breg", + "version": "0.1.0", + "license": "Apache-2.0", + "dependencies": { + "@openfn/language-common": "3.3.4", + "@registrystack/client": "0.27.0" + }, + "devDependencies": { + "@openfn/compiler": "1.3.0", + "@openfn/runtime": "1.9.5" + }, + "engines": { + "node": ">=22.12.0" + } + }, + "packages/registry-evidence": { + "name": "@openfn/language-registry-evidence", + "version": "0.1.0", + "license": "Apache-2.0", + "dependencies": { + "@openfn/language-common": "3.3.4", + "@registrystack/client": "0.27.0" + }, + "devDependencies": { + "@openfn/compiler": "1.3.0", + "@openfn/runtime": "1.9.5" + }, + "engines": { + "node": ">=22.12.0" + } + }, "packages/registry-notary": { "name": "@openfn/language-registry-notary", "version": "0.1.0", @@ -2635,6 +2980,55 @@ "node": ">=20.18.1" } }, + "packages/registry-notary/node_modules/@openfn/compiler": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@openfn/compiler/-/compiler-1.2.5.tgz", + "integrity": "sha512-X0JcfC0rPfD/RJwMGr3uNOp+YiixJ86V1JintarndCUNOUI+Not9wq6sLQSz51YCMXbMezMhbY+QZN0e7WyzNA==", + "dev": true, + "license": "ISC", + "dependencies": { + "@openfn/describe-package": "0.1.6", + "@openfn/lexicon": "^2.0.0", + "@openfn/logger": "1.1.2", + "acorn": "^8.16.0", + "ast-types": "^0.14.2", + "recast": "^0.21.5" + }, + "engines": { + "node": ">=16", + "pnpm": ">=7" + } + }, + "packages/registry-notary/node_modules/@openfn/language-common": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@openfn/language-common/-/language-common-3.3.3.tgz", + "integrity": "sha512-i5uaM0inXROpU25C2zXV6h4rzEq19oMSGmg939vu3JBD5FxopyWR6ZMXAEho5EiFv7QPIHGueTdQbCQg+V5fWg==", + "license": "LGPL-3.0-or-later", + "dependencies": { + "ajv": "^8.20.0", + "csv-parse": "^5.4.0", + "csvtojson": "^2.0.13", + "date-fns": "^2.25.0", + "http-status-codes": "^2.3.0", + "jsonpath-plus": "^10.3.0", + "lodash": "^4.18.1", + "undici": "^7.24.7" + } + }, + "packages/registry-notary/node_modules/@openfn/runtime": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/@openfn/runtime/-/runtime-1.9.3.tgz", + "integrity": "sha512-m4HeHYdWONWHjfayhKne7CgTIGx6l+p7MANevVIbrZs4dmFJb+g+DcHcikfRvW7S9sA2xBFnXK7FyMl/QjD7AQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "@openfn/logger": "1.1.2", + "fast-safe-stringify": "^2.1.1", + "json-stream-stringify": "^3.1.6", + "semver": "^7.7.4", + "source-map": "^0.7.6" + } + }, "packages/registry-relay": { "name": "@openfn/language-registry-relay", "version": "0.1.0", @@ -2649,6 +3043,55 @@ "engines": { "node": ">=20.18.1" } + }, + "packages/registry-relay/node_modules/@openfn/compiler": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@openfn/compiler/-/compiler-1.2.5.tgz", + "integrity": "sha512-X0JcfC0rPfD/RJwMGr3uNOp+YiixJ86V1JintarndCUNOUI+Not9wq6sLQSz51YCMXbMezMhbY+QZN0e7WyzNA==", + "dev": true, + "license": "ISC", + "dependencies": { + "@openfn/describe-package": "0.1.6", + "@openfn/lexicon": "^2.0.0", + "@openfn/logger": "1.1.2", + "acorn": "^8.16.0", + "ast-types": "^0.14.2", + "recast": "^0.21.5" + }, + "engines": { + "node": ">=16", + "pnpm": ">=7" + } + }, + "packages/registry-relay/node_modules/@openfn/language-common": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@openfn/language-common/-/language-common-3.3.3.tgz", + "integrity": "sha512-i5uaM0inXROpU25C2zXV6h4rzEq19oMSGmg939vu3JBD5FxopyWR6ZMXAEho5EiFv7QPIHGueTdQbCQg+V5fWg==", + "license": "LGPL-3.0-or-later", + "dependencies": { + "ajv": "^8.20.0", + "csv-parse": "^5.4.0", + "csvtojson": "^2.0.13", + "date-fns": "^2.25.0", + "http-status-codes": "^2.3.0", + "jsonpath-plus": "^10.3.0", + "lodash": "^4.18.1", + "undici": "^7.24.7" + } + }, + "packages/registry-relay/node_modules/@openfn/runtime": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/@openfn/runtime/-/runtime-1.9.3.tgz", + "integrity": "sha512-m4HeHYdWONWHjfayhKne7CgTIGx6l+p7MANevVIbrZs4dmFJb+g+DcHcikfRvW7S9sA2xBFnXK7FyMl/QjD7AQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "@openfn/logger": "1.1.2", + "fast-safe-stringify": "^2.1.1", + "json-stream-stringify": "^3.1.6", + "semver": "^7.7.4", + "source-map": "^0.7.6" + } } } } diff --git a/package.json b/package.json index 1088241..45ca0b2 100644 --- a/package.json +++ b/package.json @@ -8,11 +8,18 @@ "packages/*" ], "scripts": { - "check": "npm run check --workspaces", - "test": "npm test --workspaces", - "pack:dry-run": "npm run pack:dry-run --workspaces" + "check": "npm run check:evidence && npm run check:breg && npm run check:events && npm run check:pilot", + "check:evidence": "npm run check --workspace @openfn/language-registry-evidence", + "check:breg": "npm run check --workspace @openfn/language-registry-breg", + "check:events": "node --test services/event-bridge/test/*.test.js services/destination/test/*.test.js", + "check:pilot": "node --experimental-vm-modules --no-warnings --test pilot/test/*.test.js && python3 -m unittest discover -s pilot/test -p \"*_test.py\"", + "test": "npm run check", + "pack:dry-run": "npm run pack:dry-run --workspace @openfn/language-registry-evidence && npm run pack:dry-run --workspace @openfn/language-registry-breg" + }, + "dependencies": { + "@openfn/language-http": "7.3.3" }, "engines": { - "node": ">=20.18.1" + "node": ">=22.12.0" } } diff --git a/packages/registry-breg/README.md b/packages/registry-breg/README.md new file mode 100644 index 0000000..18e7dc0 --- /dev/null +++ b/packages/registry-breg/README.md @@ -0,0 +1,132 @@ +# @openfn/language-registry-breg + +OpenFn operations over the native `breg` namespace in the exact published +`@registrystack/client@0.27.0`. The adaptor uses +`@openfn/language-common@3.3.4` and reexports its composition helpers. +It does not implement Registry authorization or an alternative HTTP client. + +## Configuration and OpenFn state + +Put the native SDK configuration in `state.configuration.breg`, or directly in +`state.configuration` for a standalone BREG job: + +```json +{ + "breg": { + "baseUrl": "https://registry.example.invalid/", + "authorization": { "static": "REPLACE_WITH_BEARER_TOKEN" }, + "requestTimeoutMilliseconds": 30000, + "maxResponseBytes": 4194304 + } +} +``` + +Alternatively use `authorization: { privateKeyJwt: { tokenEndpoint, clientId, +clientKey, audience } }`, where `clientKey` is a private JWK with `kty`, `kid`, +and `alg`. Token and Registry TLS trust roots and timeouts are independent. +The [configuration schema](./configuration-schema.json) lists the fields; +the native SDK validates supported algorithms, bounds and URLs. Keep keys and +tokens in OpenFn credentials, never event data or expressions. + +Every operation returns a state function. Pass an options object, a function +returning options, or state functions as individual option values. Nested data +is plain JSON; use a function on `data`, `values`, or `operations` to build it. +Results appear at `state.data[as]`, defaulting to `state.data.breg`: + +```js +{ branch: 'succeeded', value, etag, location, continuation, traceId } +``` + +Optional fields appear only when the SDK returned them. A read returns a Registry +Record envelope at `value`, lists return a collection, and lifecycle actions +return receipts. These remain different contracts. Successful metadata discovery +returns descriptive JSON, never opaque SDK authority handles. + +Configuration remains present between operations so `execute`, `each`, and +successive requests can authenticate. The OpenFn runtime removes configuration +from final output by default, as exercised in the compiled-job test. Direct +JavaScript callers must remove it before logging or persisting the final state. +Record inputs and outputs remain in `data`; minimize and remove them explicitly +when the next step no longer needs them. The adaptor does not log requests. + +## Operations + +| Operation | Required inputs and behavior | +| --- | --- | +| `discoverRegistry` | Optional `accessProfile`; caller-filtered metadata. | +| `getEntitySchema` | `entityIdentifier`, optional `accessProfile`. | +| `getRecord` | `route`, `recordIdentifier`. | +| `lookupRecord` | `route`, `selector`; `values` for request selectors, omitted for verified-claim selectors. Exact lookup, no list scan fallback. | +| `listRecords` | `route`; optional `top`, `filter`, `orderby`, `count`. One page only. | +| `continueList` | Exact SDK `continuation` from the previous page. | +| `createRecord` | `operationIdentifier`, `accessProfile`, `data`, `idempotencyKey`. | +| `patchRecord` | `operationIdentifier`, `accessProfile`, `recordIdentifier`, `etag`, `operations`, `idempotencyKey`. | +| `createChangeRequest` | Same as create; use the configured change-request entity's create operation to create a draft. | +| `executeLifecycleAction` | `entityIdentifier`, `route`, `recordIdentifier`, `accessProfile`, `operation`, action `ifMatch`, `idempotencyKey`; `stage` for staged review. | + +Reads and lookup accept `select`, `accessProfile`, and `format` (`json` or +`json-ld`). Lists also accept these. Creates, patches and lifecycle actions accept +`format`. Use the authored entity route, operation ID and field API names exactly +as published in metadata. Patch members use `{ op: 'replace', field: 'legalName', +value: '...' }`, not raw HTTP JSON Pointer paths. The native client constructs +and validates the HTTP patch document. + +Writes fetch caller-filtered metadata inside the operation and select an opaque +binding for the exact operation and profile. No function accepts a write URL, +serialized binding, or arbitrary action URL as authority. Immediate-action +metadata remains descriptive; this adaptor does not execute it. + +Use the direct read's exact strong ETag for patching. Lifecycle `ifMatch` is the +separate action-specific value in `value.data.request.actions`, never the record +ETag. The lifecycle helper refetches the contract and record, promotes actions +through the native client, and requires exactly one matching operation/stage and +the caller's held `ifMatch`. A changed proposal fails before any action POST. +Supported operation names are `submit_request`, `approve_request`, +`reject_request`, `request_revision`, `revise_request`, `cancel_request`, and +`apply_request`, subject to the current metadata, actor, stage and record state. + +Create a governed correction with `createChangeRequest`, read the created request +with `getRecord` to obtain its current actions, then explicitly call +`executeLifecycleAction` with `operation: 'submit_request'`. Give each mutation +its own durable idempotency key derived from the source event and operation. +Creating a draft does not approve or submit it. Review and application remain +separate actor-authorized actions. See the [create and submit job](./jobs/create-and-submit.js) +and [read and patch job](./jobs/read-and-patch.js) for synthetic templates. + +## Failures and recovery + +Operations return `branch` values `invalid_request`, `auth_failed`, `denied`, +`not_found`, `conflict`, `retryable_infrastructure`, `protocol_failed`, or `failed`. +Failures include a safe `problem` with `code`, `retryable`, and optional HTTP +`status`. Typed native `code`, `planRefusal`, `traceId`, `transportKind`, and +`tokenKind` may be present at the result level. Exception messages, response +problem prose, credentials and record values are excluded from failure results. +Check the branch before consuming a value or starting a dependent mutation. + +Neither the adaptor nor native SDK retries or follows redirects. Pagination is +explicit and bounded by server metadata. A temporary failure can leave a mutation +outcome unknown: replay the exact request with the same key and preconditions, +never invent a new key. Keep a durable source-event correlation. The lifecycle +helper refetches current state, so an already-completed action may no longer be +advertised; it then returns `denied`. Reconcile the request's current state and +source correlation before treating this as already submitted. Do not infer +success just from a missing action. Discard held metadata, ETags, continuations, +records and drafts when identity/profile changes; review drafts after registry +revision changes. + +## Verification + +```sh +npm test --workspace @openfn/language-registry-breg +npm run check --workspace @openfn/language-registry-breg +npm run pack:dry-run --workspace @openfn/language-registry-breg +``` + +The tests use the published native package against ephemeral synthetic HTTP +servers and compile/run a real OpenFn read-and-patch job. A small CommonJS bridge +copies bounded, plain JSON across the OpenFn VM boundary into the SDK realm; +opaque handles stay local. Tests cover native lookup/read/write/lifecycle HTTP +contracts, continuation, precondition refusal, failure redaction and runtime +credential removal. Private-key JWT configuration and token transport failure +are exercised; a successful live Mint exchange and real PostgreSQL governance +are integration checks outside these mock tests. diff --git a/packages/registry-breg/configuration-schema.json b/packages/registry-breg/configuration-schema.json new file mode 100644 index 0000000..2592d95 --- /dev/null +++ b/packages/registry-breg/configuration-schema.json @@ -0,0 +1,41 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Registry Stack Base Registry Engine", + "description": "Place this object in configuration.breg, or use it directly as configuration. Credentials must remain in OpenFn configuration.", + "type": "object", + "additionalProperties": false, + "required": ["baseUrl"], + "properties": { + "baseUrl": { "type": "string", "format": "uri", "description": "Base Registry deployment URL, including any reverse-proxy prefix." }, + "authorization": { + "oneOf": [ + { "type": "object", "additionalProperties": false, "required": ["static"], "properties": { "static": { "type": "string", "minLength": 1, "writeOnly": true, "format": "password", "description": "Bearer token without the Bearer prefix." } } }, + { "type": "object", "additionalProperties": false, "required": ["privateKeyJwt"], "properties": { "privateKeyJwt": { "$ref": "#/$defs/privateKeyJwt" } } } + ] + }, + "requestTimeoutMilliseconds": { "type": "integer", "minimum": 1 }, + "connectTimeoutMilliseconds": { "type": "integer", "minimum": 1 }, + "maxResponseBytes": { "type": "integer", "minimum": 1 }, + "userAgent": { "type": "string" }, + "trustedRootCertificates": { "type": "string", "description": "PEM trust roots for the Registry HTTPS connection." } + }, + "$defs": { + "privateKeyJwt": { + "type": "object", + "additionalProperties": false, + "required": ["tokenEndpoint", "clientId", "clientKey"], + "properties": { + "tokenEndpoint": { "type": "string", "format": "uri" }, + "clientId": { "type": "string", "minLength": 1 }, + "clientKey": { "type": "object", "writeOnly": true, "required": ["kty", "kid", "alg"], "properties": { "kty": { "type": "string" }, "kid": { "type": "string" }, "alg": { "type": "string" } }, "additionalProperties": true }, + "audience": { "type": "string" }, + "assertionLifetimeSeconds": { "type": "integer", "minimum": 1 }, + "refreshMarginSeconds": { "type": "integer", "minimum": 0 }, + "requestTimeoutMilliseconds": { "type": "integer", "minimum": 1 }, + "connectTimeoutMilliseconds": { "type": "integer", "minimum": 1 }, + "userAgent": { "type": "string" }, + "trustedRootCertificates": { "type": "string", "description": "PEM trust roots for the token endpoint." } + } + } + } +} diff --git a/packages/registry-breg/jobs/create-and-submit.js b/packages/registry-breg/jobs/create-and-submit.js new file mode 100644 index 0000000..b6ac3bd --- /dev/null +++ b/packages/registry-breg/jobs/create-and-submit.js @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: Apache-2.0 +// Supply the configured request entity's operation ID, route and profile in data. +// Credentials belong only in configuration.breg. +import { execute } from "@openfn/language-common"; +import { createChangeRequest, getRecord, executeLifecycleAction } from "../src/index.js"; + +execute( + createChangeRequest(state => ({ + operationIdentifier: state.data.createOperationIdentifier, + accessProfile: state.data.accessProfile, + data: state.data.requestFields, + idempotencyKey: state.data.createIdempotencyKey, + as: "draft", + })), + state => { + if (state.data.draft.branch !== "succeeded") return state; + return getRecord({ route: state.data.requestRoute, + recordIdentifier: state.data.draft.value.data.recordIdentifier, + accessProfile: state.data.accessProfile, as: "currentRequest" })(state); + }, + state => { + if (state.data.currentRequest?.branch !== "succeeded") return state; + const record = state.data.currentRequest.value; + const action = record.data.request?.actions?.find(item => item.operation === "submit_request"); + if (!action) return { ...state, data: { ...state.data, submission: { + branch: "denied", problem: { code: "action.unavailable", retryable: false }, + } } }; + return executeLifecycleAction({ + entityIdentifier: record.meta.entityTypeIdentifier, + route: state.data.requestRoute, + recordIdentifier: record.data.recordIdentifier, + accessProfile: state.data.accessProfile, + operation: "submit_request", + ifMatch: action.ifMatch, + idempotencyKey: state.data.submitIdempotencyKey, + as: "submission", + })(state); + }, +); diff --git a/packages/registry-breg/jobs/read-and-patch.js b/packages/registry-breg/jobs/read-and-patch.js new file mode 100644 index 0000000..f216d0e --- /dev/null +++ b/packages/registry-breg/jobs/read-and-patch.js @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: Apache-2.0 +// Synthetic names. Replace the route, operation and profile with your published metadata. +import { execute } from "@openfn/language-common"; +import { getRecord, patchRecord } from "../src/index.js"; + +execute( + getRecord({ route: "companies", recordIdentifier: state => state.data.recordIdentifier, + accessProfile: "company-writer", as: "current" }), + state => { + if (state.data.current.branch !== "succeeded") return state; + return patchRecord({ operationIdentifier: "records.company.patch", accessProfile: "company-writer", + recordIdentifier: state.data.recordIdentifier, etag: state.data.current.etag, + idempotencyKey: state.data.idempotencyKey, + operations: [{ op: "replace", field: "legalName", value: state.data.legalName }], as: "updated" })(state); + }, +); diff --git a/packages/registry-breg/package.json b/packages/registry-breg/package.json new file mode 100644 index 0000000..5e7a228 --- /dev/null +++ b/packages/registry-breg/package.json @@ -0,0 +1,33 @@ +{ + "name": "@openfn/language-registry-breg", + "version": "0.1.0", + "description": "OpenFn operations for Registry Stack Base Registry Engine.", + "license": "Apache-2.0", + "type": "module", + "main": "./src/index.js", + "exports": { + ".": "./src/index.js" + }, + "files": [ + "src", + "jobs", + "configuration-schema.json", + "README.md" + ], + "scripts": { + "test": "node --experimental-vm-modules --no-warnings --test test/*.test.js", + "check": "node --check src/index.js && node --check src/native.cjs && node --check jobs/read-and-patch.js && node --check jobs/create-and-submit.js && npm test", + "pack:dry-run": "npm pack --dry-run" + }, + "engines": { + "node": ">=22.12.0" + }, + "dependencies": { + "@openfn/language-common": "3.3.4", + "@registrystack/client": "0.27.0" + }, + "devDependencies": { + "@openfn/compiler": "1.3.0", + "@openfn/runtime": "1.9.5" + } +} diff --git a/packages/registry-breg/src/index.js b/packages/registry-breg/src/index.js new file mode 100644 index 0000000..c246f6d --- /dev/null +++ b/packages/registry-breg/src/index.js @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: Apache-2.0 +import { createRequire } from "node:module"; +export * from "@openfn/language-common"; + +const require = createRequire(import.meta.url); +const { BaseRegistryClient, BaseRegistryClientError } = require("./native.cjs"); + +class OperationFailure extends Error { + constructor(branch, code) { + super("Base Registry operation did not complete"); + this.branch = branch; + this.code = code; + } +} + +/** Return caller-filtered descriptive metadata. Opaque write bindings stay local. */ +export function discoverRegistry(options = {}) { + return operation(options, async (client, input) => document( + await client.registryMetadata(input.accessProfile), + )); +} + +export function getEntitySchema(options = {}) { + return operation(options, async (client, input) => document( + await client.entitySchema(required(input.entityIdentifier), input.accessProfile), + )); +} + +export function getRecord(options = {}) { + return operation(options, (client, input) => client.getRecord( + required(input.route), required(input.recordIdentifier), readOptions(input), + )); +} + +export function lookupRecord(options = {}) { + return operation(options, (client, input) => client.lookupRecord( + required(input.route), required(input.selector), input.values, readOptions(input), + )); +} + +/** Fetch one bounded page. Continue explicitly with its SDK continuation. */ +export function listRecords(options = {}) { + return operation(options, (client, input) => client.listRecords( + required(input.route), pick(input, ["select", "accessProfile", "format", "top", "filter", "orderby", "count"]), + )); +} + +export function continueList(options = {}) { + return operation(options, (client, input) => client.continueList(input.continuation)); +} + +export function createRecord(options = {}) { + return operation(options, async (client, input) => { + const key = required(input.idempotencyKey); + const identifier = required(input.operationIdentifier); + const profile = required(input.accessProfile); + const metadata = await client.registryContract(profile); + return client.createRecord(metadata.selectCreate(identifier, profile), input.data, key, input.format); + }); +} + +/** Create a draft in the configured change-request entity. Submission is a separate lifecycle action. */ +export function createChangeRequest(options = {}) { + return createRecord(options); +} + +export function patchRecord(options = {}) { + return operation(options, async (client, input) => { + const key = required(input.idempotencyKey); + const etag = required(input.etag); + const identifier = required(input.operationIdentifier); + const profile = required(input.accessProfile); + const recordIdentifier = required(input.recordIdentifier); + const metadata = await client.registryContract(profile); + return client.patchRecord(metadata.selectPatch(identifier, profile), recordIdentifier, + etag, input.operations, key, input.format); + }); +} + +/** Execute exactly one currently advertised action using a caller's held lifecycle precondition. */ +export function executeLifecycleAction(options = {}) { + return operation(options, async (client, input) => { + const key = required(input.idempotencyKey); + const ifMatch = required(input.ifMatch); + const profile = required(input.accessProfile); + const entity = required(input.entityIdentifier); + const route = required(input.route); + const recordIdentifier = required(input.recordIdentifier); + const requestedOperation = required(input.operation); + const metadata = await client.registryContract(profile); + const authority = metadata.selectLifecycle(entity, profile); + const record = await client.getRecord(route, recordIdentifier, { accessProfile: profile, format: input.format ?? "json" }); + const actions = client.lifecycleActions(authority, record.value, input.format); + const selected = actions.filter((action) => action.operation === requestedOperation + && (action.stage ?? null) === (input.stage ?? null)); + if (selected.length !== 1) throw new OperationFailure("denied", "action.unavailable"); + const action = selected[0]; + const advertised = record.value.data.request?.actions ?? []; + if (!advertised.some((item) => item.href === action.href && item.operation === action.operation + && (item.stage ?? null) === (action.stage ?? null) && item.ifMatch === ifMatch)) { + throw new OperationFailure("conflict", "action.precondition_changed"); + } + return client.executeLifecycleAction(action, key); + }); +} + +function operation(options, invoke) { + return async (state) => { + let input; + let result; + try { + const supplied = typeof options === "function" ? options(state) : options; + if (!supplied || typeof supplied !== "object" || Array.isArray(supplied)) { + throw new OperationFailure("invalid_request", "request.invalid"); + } + // State functions are resolved at option boundaries; record graphs go intact to the SDK validator. + input = Object.fromEntries(Object.entries(supplied).map(([key, value]) => [key, + typeof value === "function" ? value(state) : value])); + const configuration = state.configuration?.breg ?? state.configuration; + const client = new BaseRegistryClient(configuration); + const outcome = await invoke(client, input); + result = { branch: "succeeded", ...pick(outcome, ["value", "etag", "location", "continuation", "traceId"]) }; + } catch (error) { + result = failure(error); + } + const as = typeof input?.as === "string" && input.as.length ? input.as : "breg"; + // OpenFn owns final credential removal. Removing configuration here breaks execute/each composition. + return { ...state, data: { ...state.data, [as]: result } }; + }; +} + +function failure(error) { + if (error instanceof OperationFailure) { + return { branch: error.branch, problem: { code: error.code, retryable: false } }; + } + if (!(error instanceof BaseRegistryClientError)) { + return { branch: "failed", problem: { code: "operation.failed", retryable: false } }; + } + const status = Number.isSafeInteger(error.status) ? error.status : 0; + let branch = "failed"; + if (error.kind === "configuration" || error.kind === "invalid_request") branch = "invalid_request"; + else if (error.kind === "transport" || (error.kind === "token" && error.tokenKind === "transport") + || (error.kind === "problem" && (status === 429 || status >= 500))) branch = "retryable_infrastructure"; + else if (error.kind === "token" || status === 401) branch = "auth_failed"; + else if (status === 403 || error.kind === "metadata_selection") branch = "denied"; + else if (status === 404 || error.kind === "not_found") branch = "not_found"; + else if (status === 409 || status === 412) branch = "conflict"; + else if (error.kind === "protocol") branch = "protocol_failed"; + return { + branch, + problem: { code: `breg.${error.kind}`, status, retryable: branch === "retryable_infrastructure" }, + // Only typed diagnostics, never the SDK error message, response body, or caller inputs. + ...pick(error, ["code", "planRefusal", "traceId", "transportKind", "tokenKind"]), + }; +} + +function required(value) { + if (typeof value !== "string" || !value.trim()) throw new OperationFailure("invalid_request", "request.required"); + return value; +} + +function pick(value, keys) { + return Object.fromEntries(keys.filter((key) => value[key] !== undefined).map((key) => [key, value[key]])); +} + +function readOptions(input) { return pick(input, ["select", "accessProfile", "format"]); } + +function document(outcome) { + try { return { ...outcome, value: JSON.parse(outcome.body.toString("utf8")) }; } + catch { throw new OperationFailure("protocol_failed", "response.invalid_json"); } +} diff --git a/packages/registry-breg/src/native.cjs b/packages/registry-breg/src/native.cjs new file mode 100644 index 0000000..175a0fb --- /dev/null +++ b/packages/registry-breg/src/native.cjs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: Apache-2.0 +// OpenFn expressions run in a VM realm. Normalize plain JSON into the SDK's +// realm while retaining its bounded, accessor-free input contract. +const { types: { isProxy } } = require("node:util"); +const { breg } = require("@registrystack/client"); +const { BaseRegistryClientError } = breg; +function copy(value, kind = "invalid_request") { + const active = new Set(); + let nodes = 0; + let bytes = 0; + const invalid = () => { throw new BaseRegistryClientError({ kind, message: "Base Registry input is invalid" }); }; + const string = text => { bytes += Buffer.byteLength(text); if (bytes > 4 * 1024 * 1024) invalid(); return text; }; + function visit(item, depth) { + if (++nodes > 100000 || depth > 128) invalid(); + if (item === null || typeof item === "boolean") return item; + if (typeof item === "string") return string(item); + if (typeof item === "number" && Number.isFinite(item)) return item; + if (!item || typeof item !== "object" || isProxy(item) || active.has(item)) invalid(); + const array = Array.isArray(item); + const prototype = Object.getPrototypeOf(item); + // Intrinsic Object.prototype has a null parent in each realm. Arrays + // additionally have one Array.prototype layer. Custom classes are refused. + if (array) { + if (!prototype || Object.getOwnPropertyDescriptor(prototype, "constructor")?.value?.name !== "Array" + || Object.getPrototypeOf(Object.getPrototypeOf(prototype)) !== null) invalid(); + } else if (prototype !== null && (Object.getPrototypeOf(prototype) !== null + || Object.getOwnPropertyDescriptor(prototype, "constructor")?.value?.name !== "Object")) invalid(); + active.add(item); + const result = array ? [] : {}; + let count = 0; + for (const key of Reflect.ownKeys(item)) { + if (typeof key !== "string") invalid(); + const descriptor = Object.getOwnPropertyDescriptor(item, key); + if (!descriptor || !Object.hasOwn(descriptor, "value")) invalid(); + if (array && key === "length") continue; + if (!descriptor.enumerable) invalid(); + string(key); + if (array && (!Number.isSafeInteger(Number(key)) || Number(key) < 0 || String(Number(key)) !== key || Number(key) >= item.length)) invalid(); + Object.defineProperty(result, key, { value: visit(descriptor.value, depth + 1), enumerable: true, writable: true, configurable: true }); + count++; + } + if (array && count !== item.length) invalid(); + active.delete(item); + return result; + } + return visit(value, 0); +} +class BaseRegistryClient extends breg.BaseRegistryClient { + constructor(config) { super(copy(config, "configuration")); } +} +for (const [method, indexes] of [ + ["getRecord", [2]], ["listRecords", [1]], ["continueList", [0]], + ["lookupRecord", [2, 3]], ["createRecord", [1]], ["patchRecord", [3]], ["lifecycleActions", [1]], +]) { + BaseRegistryClient.prototype[method] = function (...args) { + for (const index of indexes) if (args[index] !== undefined) args[index] = copy(args[index]); + return breg.BaseRegistryClient.prototype[method].apply(this, args); + }; +} +module.exports = { BaseRegistryClient, BaseRegistryClientError }; diff --git a/packages/registry-breg/test/breg.test.js b/packages/registry-breg/test/breg.test.js new file mode 100644 index 0000000..21373c1 --- /dev/null +++ b/packages/registry-breg/test/breg.test.js @@ -0,0 +1,284 @@ +import assert from "node:assert/strict"; +import { generateKeyPairSync } from "node:crypto"; +import { readFileSync } from "node:fs"; +import http from "node:http"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import compile from "@openfn/compiler"; +import run from "@openfn/runtime"; +import { execute, discoverRegistry, getEntitySchema, getRecord, lookupRecord, listRecords, continueList, createRecord, patchRecord, createChangeRequest, executeLifecycleAction } from "../src/index.js"; +import { ID, ETAG, ACTION_ETAG, TRACE, PROFILE, metadata, record, receipt } from "./fixtures.js"; + +async function server(handler) { + const requests = []; + const httpServer = http.createServer(async (req, res) => { + let body = ""; + for await (const chunk of req) body += chunk; + requests.push({ url: req.url, method: req.method, headers: req.headers, body }); + res.setHeader("traceparent", `00-${TRACE}-00f067aa0ba902b7-01`); + res.setHeader("content-type", "application/json"); + res.setHeader("cache-control", "no-store"); + res.setHeader("vary", "authorization, accept"); + res.setHeader("link", '; rel="profile", ; rel="describedby"'); + await handler(req, res); + }); + await new Promise(resolve => httpServer.listen(0, "127.0.0.1", resolve)); + return { requests, baseUrl: `http://127.0.0.1:${httpServer.address().port}/tenant`, close: () => new Promise(resolve => httpServer.close(resolve)) }; +} +function state(baseUrl) { return { configuration: { breg: { baseUrl, authorization: { static: "synthetic-breg-token" } } }, data: {} }; } +function regular(req, res) { + if (req.url.includes("/v1/registry")) return res.end(JSON.stringify(metadata())); + if (req.url.includes("/v1/schemas/")) return res.end('{"type":"object"}'); + if (req.url.includes("/actions/")) { res.removeHeader("link"); return res.end(JSON.stringify(receipt())); } + if (!req.url.includes("lookup")) res.setHeader("etag", ETAG); + if (req.method === "POST" && !req.url.includes("lookup")) { + res.statusCode = 201; + res.setHeader("location", `/tenant/v1/records/companies/${ID}`); + } + res.end(JSON.stringify(record())); +} +const write = { operationIdentifier: "records.company.create", accessProfile: PROFILE, idempotencyKey: "synthetic-create-001", data: { legalName: "Example Ltd" } }; +const lifecycle = { entityIdentifier: "company", route: "companies", recordIdentifier: ID, accessProfile: PROFILE, operation: "submit_request", ifMatch: ACTION_ETAG, idempotencyKey: "synthetic-submit-001" }; + +test("native discovery, exact lookup, read, and schema preserve composition configuration", async () => { + const stub = await server(regular); + try { + const initial = state(stub.baseUrl); + const result = await execute(discoverRegistry({ accessProfile: PROFILE, as: "metadata" }), lookupRecord({ route: "companies", selector: "by-name", values: { legalName: "Example Ltd" }, accessProfile: PROFILE, as: "lookup" }), getRecord({ route: "companies", recordIdentifier: ID, as: "record" }), getEntitySchema({ entityIdentifier: "company", as: "schema" }))(initial); + for (const key of ["metadata", "lookup", "record", "schema"]) assert.equal(result.data[key].branch, "succeeded", JSON.stringify(result.data[key])); + assert.equal(result.configuration, initial.configuration); + assert.deepEqual(JSON.parse(stub.requests[1].body), { selector: "by-name", values: { legalName: "Example Ltd" } }); + assert.ok(stub.requests.every(req => req.headers.authorization === "Bearer synthetic-breg-token")); + } finally { await stub.close(); } +}); + +test("metadata-selected create and patch carry caller keys and exact record ETag", async () => { + const stub = await server(regular); + try { + const result = await execute(createRecord({ ...write, as: "created" }), patchRecord(s => ({ operationIdentifier: "records.company.patch", accessProfile: PROFILE, recordIdentifier: ID, etag: s.data.created.etag, idempotencyKey: "synthetic-patch-001", operations: [{ op: "replace", field: "legalName", value: "Renamed Ltd" }] })))(state(stub.baseUrl)); + assert.equal(result.data.created.branch, "succeeded", JSON.stringify(result.data.created)); + assert.equal(result.data.breg.branch, "succeeded", JSON.stringify(result.data.breg)); + assert.equal(stub.requests[1].headers["idempotency-key"], write.idempotencyKey); + assert.deepEqual(JSON.parse(stub.requests[1].body), { data: write.data }); + const patch = stub.requests[3]; + assert.equal(patch.headers["if-match"], ETAG); + assert.equal(patch.headers["content-type"], "application/json-patch+json"); + assert.deepEqual(JSON.parse(patch.body), [{ op: "replace", path: "/data/legalName", value: "Renamed Ltd" }]); + assert.ok(!JSON.stringify(result.data).includes("BRegCreateBinding")); + } finally { await stub.close(); } +}); + +test("missing keys/preconditions and arbitrary operation URLs cannot write", async () => { + const stub = await server(regular); + try { + for (const op of [createRecord({ ...write, idempotencyKey: undefined }), patchRecord({ ...write, etag: undefined }), executeLifecycleAction({ ...lifecycle, ifMatch: undefined })]) { + assert.equal((await op(state(stub.baseUrl))).data.breg.branch, "invalid_request"); + } + assert.equal(stub.requests.length, 0); + const result = await createRecord({ ...write, operationIdentifier: "https://attacker.invalid/write" })(state(stub.baseUrl)); + assert.equal(result.data.breg.branch, "denied"); + assert.equal(stub.requests.length, 1); + const draft = await createChangeRequest(write)(state(stub.baseUrl)); + assert.equal(draft.data.breg.branch, "succeeded"); + } finally { await stub.close(); } +}); + +test("native lifecycle promotion executes exact action and refuses a changed precondition", async () => { + const stub = await server((req, res) => { + if (req.method === "GET" && req.url.includes(`/companies/${ID}`)) { res.setHeader("etag", ETAG); return res.end(JSON.stringify(record(true))); } + regular(req, res); + }); + try { + const result = await executeLifecycleAction(lifecycle)(state(stub.baseUrl)); + assert.equal(result.data.breg.branch, "succeeded", JSON.stringify(result.data.breg)); + assert.equal(stub.requests[2].headers["if-match"], ACTION_ETAG); + assert.equal(stub.requests[2].headers["idempotency-key"], lifecycle.idempotencyKey); + assert.deepEqual(JSON.parse(stub.requests[2].body), {}); + const stale = await executeLifecycleAction({ ...lifecycle, ifMatch: ETAG })(state(stub.baseUrl)); + assert.equal(stale.data.breg.branch, "conflict"); + assert.equal(stub.requests.filter(req => req.method === "POST").length, 1); + } finally { await stub.close(); } +}); + +test("one explicit page and continuation preserve native cursor bindings", async () => { + const stub = await server((req, res) => res.end(JSON.stringify({ items: [record().data], pageInfo: { nextCursor: req.url.includes("skiptoken") ? null : "synthetic_cursor" }, meta: record().meta }))); + try { + const first = await listRecords({ route: "companies", top: 1, accessProfile: PROFILE })(state(stub.baseUrl)); + assert.equal(first.data.breg.branch, "succeeded", JSON.stringify(first.data.breg)); + assert.equal(stub.requests.length, 1); + const second = await continueList({ continuation: first.data.breg.continuation })(first); + assert.equal(second.data.breg.branch, "succeeded", JSON.stringify(second.data.breg)); + assert.equal(stub.requests.length, 2); + const params = new URL(stub.requests[1].url, stub.baseUrl).searchParams; + assert.equal(params.get("$skiptoken"), "synthetic_cursor"); + assert.equal(params.has("$top"), false); + } finally { await stub.close(); } +}); + +test("safe stable failures distinguish not found, denied, conflict and temporary service errors without retries", async () => { + for (const [status, code, branch] of [[404, "resource.not_found", "not_found"], [403, "auth.forbidden", "denied"], [409, "mutation.conflict", "conflict"], [503, "source.unavailable", "retryable_infrastructure"]]) { + const stub = await server((_req, res) => { + res.statusCode = status; + res.setHeader("content-type", "application/problem+json"); + res.end(JSON.stringify({ type: status === 503 ? "https://id.registrystack.org/problems/registry-breg/source/unavailable" : "about:blank", title: status === 503 ? "Service Unavailable" : "secret-canary", status, code, traceId: TRACE, detail: status === 503 ? "The Registry data service is unavailable." : "secret-canary" })); + }); + try { + const result = await getRecord({ route: "companies", recordIdentifier: ID })(state(stub.baseUrl)); + assert.equal(result.data.breg.branch, branch, JSON.stringify(result.data.breg)); + assert.equal(stub.requests.length, 1); + assert.ok(!JSON.stringify(result.data.breg).includes("secret-canary")); + } finally { await stub.close(); } + } +}); + +test("privateKeyJwt configuration reaches native validation without disclosing keys", async () => { + const { privateKey } = generateKeyPairSync("ec", { namedCurve: "prime256v1" }); + const key = { ...privateKey.export({ format: "jwk" }), alg: "ES256", kid: "synthetic-key" }; + const configured = { configuration: { baseUrl: "http://127.0.0.1:1", authorization: { privateKeyJwt: { tokenEndpoint: "https://127.0.0.1:1/token", clientId: "synthetic-client", clientKey: key, requestTimeoutMilliseconds: 100, connectTimeoutMilliseconds: 100 } } }, data: {} }; + const result = await discoverRegistry()(configured); + assert.equal(result.data.breg.branch, "retryable_infrastructure", JSON.stringify(result.data.breg)); + assert.ok(!JSON.stringify(result.data).includes(key.d)); +}); + +test("compiled OpenFn job composes native reads and writes and runtime removes credentials", async () => { + const stub = await server(regular); + try { + const { code } = compile(readFileSync(new URL("../jobs/read-and-patch.js", import.meta.url), "utf8")); + const result = await run({ workflow: { steps: [{ id: "breg", expression: code }], start: "breg" }, options: { start: "breg" } }, + { ...state(stub.baseUrl), data: { recordIdentifier: ID, idempotencyKey: "synthetic-job-patch-001", legalName: "Renamed Ltd" } }, + { linker: { modules: { "@openfn/language-common": { path: fileURLToPath(new URL("../../../node_modules/@openfn/language-common", import.meta.url)) }, "../src/index.js": { path: fileURLToPath(new URL("..", import.meta.url)) } }, cacheKey: `breg-${process.pid}` } }); + assert.equal(result.errors, undefined, JSON.stringify(result.errors)); + assert.equal(result.data.updated.branch, "succeeded", JSON.stringify(result.data)); + assert.equal("configuration" in result, false); + assert.ok(!JSON.stringify(result).includes("synthetic-breg-token")); + } finally { await stub.close(); } +}); + +test("malformed plain-data graphs fail without accessing properties or sending mutations", async () => { + const stub = await server(regular); + let read = false; + const accessor = {}; + Object.defineProperty(accessor, "legalName", { enumerable: true, get() { read = true; throw new Error("secret-accessor-canary"); } }); + const cyclic = {}; cyclic.self = cyclic; + try { + for (const data of [accessor, cyclic, new Date(), new Proxy({}, {})]) { + const result = await createRecord({ ...write, data })(state(stub.baseUrl)); + assert.equal(result.data.breg.branch, "invalid_request"); + assert.ok(!JSON.stringify(result.data.breg).includes("secret-accessor-canary")); + } + assert.equal(read, false); + assert.equal(stub.requests.some(req => req.method !== "GET"), false); + } finally { await stub.close(); } +}); + +test("record action URLs cannot create authority outside metadata", async () => { + const stub = await server((req, res) => { + if (req.url.includes(`/companies/${ID}`)) { + const body = record(true); + body.data.request.actions[0].href = "https://attacker.invalid/mutate"; + return res.end(JSON.stringify(body)); + } + regular(req, res); + }); + try { + const result = await executeLifecycleAction(lifecycle)(state(stub.baseUrl)); + assert.notEqual(result.data.breg.branch, "succeeded"); + assert.equal(stub.requests.length, 2); + assert.equal(stub.requests.some(req => req.method !== "GET"), false); + } finally { await stub.close(); } +}); + +test("redirects and malformed success responses fail closed without automatic follow-up", async () => { + for (const redirect of [true, false]) { + const stub = await server((_req, res) => { + if (redirect) { res.statusCode = 302; res.setHeader("location", "/tenant/elsewhere"); } + res.end("secret-malformed-response-canary"); + }); + try { + const result = await getRecord({ route: "companies", recordIdentifier: ID })(state(stub.baseUrl)); + assert.notEqual(result.data.breg.branch, "succeeded"); + assert.equal(stub.requests.length, 1); + assert.ok(!JSON.stringify(result.data.breg).includes("secret-malformed-response-canary")); + } finally { await stub.close(); } + } +}); + +test("compiled governed draft creation and explicit submission keep separate keys", async () => { + const stub = await server((req, res) => { + if (req.url.includes("/v1/registry") || req.url.includes("/actions/")) return regular(req, res); + res.setHeader("etag", ETAG); + if (req.method === "POST") { res.statusCode = 201; res.setHeader("location", `/tenant/v1/records/companies/${ID}`); } + const value = record(req.method !== "POST"); + res.end(JSON.stringify(value)); + }); + try { + const { code } = compile(readFileSync(new URL("../jobs/create-and-submit.js", import.meta.url), "utf8")); + const result = await run({ workflow: { steps: [{ id: "breg", expression: code }], start: "breg" }, options: { start: "breg" } }, + { ...state(stub.baseUrl), data: { createOperationIdentifier: "records.company.create", accessProfile: PROFILE, requestFields: { legalName: "Example Ltd" }, createIdempotencyKey: "synthetic-draft-001", submitIdempotencyKey: "synthetic-draft-submit-001", requestRoute: "companies" } }, + { linker: { modules: { "@openfn/language-common": { path: fileURLToPath(new URL("../../../node_modules/@openfn/language-common", import.meta.url)) }, "../src/index.js": { path: fileURLToPath(new URL("..", import.meta.url)) } }, cacheKey: `breg-submit-${process.pid}` } }); + assert.equal(result.errors, undefined, JSON.stringify(result.errors)); + assert.equal(result.data.submission?.branch, "succeeded", JSON.stringify(result.data)); + assert.equal("configuration" in result, false); + assert.deepEqual(stub.requests.filter(req => req.method === "POST").map(req => req.headers["idempotency-key"]), ["synthetic-draft-001", "synthetic-draft-submit-001"]); + } finally { await stub.close(); } +}); + +test("stale patch ETag conflicts without overwriting or acquiring a replacement precondition", async () => { + let current = record(); + let currentEtag = ETAG; + let committedWrites = 0; + const stub = await server((req, res) => { + if (req.url.includes("/v1/registry")) return res.end(JSON.stringify(metadata())); + if (req.method === "PATCH") { + if (req.headers["if-match"] !== currentEtag) { + res.statusCode = 412; + res.setHeader("content-type", "application/problem+json"); + return res.end(JSON.stringify({ + type: "https://id.registrystack.org/problems/registry-breg/precondition/failed", + title: "Precondition Failed", status: 412, code: "precondition.failed", + detail: "The mutation precondition failed.", traceId: TRACE, + })); + } + const operations = JSON.parse(stub.requests.at(-1).body); + assert.deepEqual(operations.map(operation => [operation.op, operation.path]), [["replace", "/data/legalName"]]); + current = { ...current, data: { ...current.data, revisionIdentifier: "2", domainData: { legalName: operations[0].value } } }; + currentEtag = '"breg-record-000000000002"'; + committedWrites++; + } + res.setHeader("etag", currentEtag); + res.end(JSON.stringify(current)); + }); + try { + const initial = await getRecord({ route: "companies", recordIdentifier: ID, accessProfile: PROFILE })(state(stub.baseUrl)); + assert.equal(initial.data.breg.branch, "succeeded"); + const heldEtag = initial.data.breg.etag; + const patch = { + operationIdentifier: "records.company.patch", accessProfile: PROFILE, + recordIdentifier: ID, etag: heldEtag, + operations: [{ op: "replace", field: "legalName", value: "Accepted update" }], + idempotencyKey: "synthetic-etag-first-001", + }; + const accepted = await patchRecord(patch)(initial); + assert.equal(accepted.data.breg.branch, "succeeded", JSON.stringify(accepted.data.breg)); + assert.notEqual(accepted.data.breg.etag, heldEtag); + const rejected = await patchRecord({ ...patch, + operations: [{ op: "replace", field: "legalName", value: "Rejected overwrite" }], + idempotencyKey: "synthetic-etag-stale-002", + })(accepted); + assert.equal(rejected.data.breg.branch, "conflict", JSON.stringify(rejected.data.breg)); + assert.equal(rejected.data.breg.problem.status, 412); + assert.equal(rejected.data.breg.code, "precondition.failed"); + assert.equal(rejected.data.breg.problem.retryable, false); + assert.equal(committedWrites, 1); + assert.deepEqual(current.data.domainData, { legalName: "Accepted update" }); + assert.equal(current.data.revisionIdentifier, "2"); + assert.deepEqual(stub.requests.map(request => request.method), ["GET", "GET", "PATCH", "GET", "PATCH"]); + assert.ok(stub.requests.filter(request => request.method === "GET").slice(1).every(request => request.url.includes("/v1/registry"))); + const attempts = stub.requests.filter(request => request.method === "PATCH"); + assert.deepEqual(attempts.map(request => request.headers["if-match"]), [heldEtag, heldEtag]); + assert.deepEqual(attempts.map(request => request.headers["idempotency-key"]), ["synthetic-etag-first-001", "synthetic-etag-stale-002"]); + const final = await getRecord({ route: "companies", recordIdentifier: ID, accessProfile: PROFILE })(rejected); + assert.equal(final.data.breg.branch, "succeeded"); + assert.deepEqual(final.data.breg.value.data.domainData, { legalName: "Accepted update" }); + } finally { await stub.close(); } +}); diff --git a/packages/registry-breg/test/fixtures.js b/packages/registry-breg/test/fixtures.js new file mode 100644 index 0000000..35933dd --- /dev/null +++ b/packages/registry-breg/test/fixtures.js @@ -0,0 +1,43 @@ +// Synthetic HTTP fixtures follow registry-breg-client/tests/write_http_boundary.rs. +export const ID = "00000000-0000-4000-8000-000000000001"; +export const ETAG = '"breg-record-000000000001"'; +export const ACTION_ETAG = '"breg-action-hmac-sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"'; +export const DIGEST = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +export const TRACE = "4bf92f3577b34da6a3ce929d0e0e4736"; +export const PROFILE = "company-writer"; +const field = { id: "legal-name", apiName: "legalName", label: "Legal name", schema: { type: "string" }, required: true, nullable: true, readOnly: false, removable: true }; +function operation(id, method, path, kind, request, create = [], patch = []) { + return { id, method, path, operation: kind, sourceEntity: "company", responseEntity: "company", accessProfile: PROFILE, + requiredCapabilities: kind === "submit_request" ? ["change_request_lifecycle"] : [], entityLabel: "Companies", + identifier: { apiName: "id", location: "envelope" }, titleFields: ["legal-name"], fields: [field], readableFields: ["legal-name"], + createWritableFields: create, patchWritableFields: patch, selectors: [], query: null, request: { fieldNames: "api", queryParameters: [], ...request } }; +} +export function metadata() { + return { id: "business-registry", version: "1.2.3", revision: `sha256:${"a".repeat(64)}`, metadataVersion: "1", + entities: [{ id: "company", datasetIdentifier: "legal-entities", route: "companies", operations: [ + { operation: "create", accessProfile: PROFILE }, { operation: "patch", accessProfile: PROFILE }, { operation: "submit_request", accessProfile: PROFILE }, + ], readableFields: ["legal-name"], schema: "/v1/schemas/company" }], + operations: [ + operation("records.company.create", "POST", "/v1/records/companies", "create", { + body: "data_envelope", contentType: "application/json", idempotencyKeyRequired: true, mutationSemantics: "direct", + schema: { type: "object", additionalProperties: false, required: ["data"], properties: { data: { type: "object" } } }, + }, ["legal-name"]), + operation("records.company.patch", "PATCH", "/v1/records/companies/{record_id}", "patch", { + body: "json_patch", contentType: "application/json-patch+json", patchPathPrefix: "/data/", patchOperations: ["add", "replace", "remove", "test"], + removeSemantics: "set_null", ifMatchRequired: true, idempotencyKeyRequired: true, mutationSemantics: "direct", + schema: { type: "array", items: { oneOf: [{ type: "object" }] } }, + }, [], ["legal-name"]), + operation("records.company.request.submit", "POST", "/v1/records/companies/{record_id}/actions/submit", "submit_request", { + body: "change_request_action", contentType: "application/json", ifMatchRequired: true, idempotencyKeyRequired: true, mutationSemantics: "change_request_lifecycle", + schema: { $schema: "https://json-schema.org/draft/2020-12/schema", type: "object", additionalProperties: false, properties: {} }, + }), + ], + }; +} +export function record(lifecycle = false) { + return { data: { recordIdentifier: ID, revisionIdentifier: lifecycle ? "7" : "1", domainData: { legalName: "Example Ltd" }, snapshot: `breg1_${ID}`, + ...(lifecycle ? { request: { bregState: "draft", proposalVersion: 7, effectDigest: DIGEST, editable: true, + actions: [{ operation: "submit_request", method: "POST", href: `/v1/records/companies/${ID}/actions/submit?accessProfile=${PROFILE}`, ifMatch: ACTION_ETAG }] } } : {}), + }, meta: { registryIdentifier: "business-registry", datasetIdentifier: "legal-entities", entityTypeIdentifier: "company" } }; +} +export function receipt() { return { id: ID, revision: 8, snapshot: `breg1_${ID}`, request: { bregState: "submitted", proposalVersion: 7, effectDigest: DIGEST, application: null } }; } diff --git a/packages/registry-evidence/README.md b/packages/registry-evidence/README.md new file mode 100644 index 0000000..84e05a7 --- /dev/null +++ b/packages/registry-evidence/README.md @@ -0,0 +1,178 @@ +# OpenFn Registry Evidence Adaptor + +OpenFn helpers for requesting and verifying minimized, signed Registry Stack +Evidence assertions with Registry Stack's own relying-party client. + +When this repository is used as `OPENFN_ADAPTORS_REPO`, this package is loaded +as: + +```text +@openfn/language-registry-evidence@local +``` + +## Client packaging + +This package pins `@registrystack/client` to **0.27.0** and uses its `evidence` +namespace. Supported native targets are macOS arm64, Linux amd64 glibc, and +Linux arm64 glibc. Alpine/musl is unsupported. Use a glibc OpenFn worker image. + +The upstream client is a Node-API native binding around the Rust Evidence +client and verifier. It is not a WebAssembly build. It owns request preparation, +fresh nonce generation, HTTP transport, response bounds, pinned-key trust, +revocation, signature verification, time checks, and policy comparison. This +adaptor only resolves OpenFn inputs, maps typed client failures to workflow +branches, redacts selectors and credentials, and shapes the verified result. + +## Configure + +Use `configuration.evidence` with the SDK's current configuration: + +```json +{ + "evidence": { + "baseUrl": "https://evidence.example.gov", + "trustedJwks": { "keys": [] }, + "revokedKeyIds": [], + "token": { "static": "" } + } +} +``` + +Populate `trustedJwks.keys` with reviewed signing keys before use. For a +private-key-JWT client-credentials exchange, replace `token` with: + +```json +{ + "privateKeyJwt": { + "tokenEndpoint": "https://issuer.example.gov/token", + "clientId": "openfn-pilot", + "clientKey": { "kty": "EC", "crv": "P-256", "kid": "", "x": "", "y": "", "d": "" } + } +} +``` + +Store private material in the OpenFn credential, never job source or state.data. +The native SDK validates the key, performs the exchange, and owns token caching. +Optional SDK timeout, response-bound, and private-CA fields pass through intact. +Exactly one token source is allowed. Existing flat credentials +(`evidence_base_url`, `token`, `evidence_verification_jwks`, +`evidence_revoked_key_ids`, plus optional `evidence_trusted_root_certificates` +and `evidence_max_response_bytes`) remain a legacy shorthand. Do not combine +flat fields with `evidence`. + +The JWKS and revocation list are relying-party trust configuration. Verification +does not fetch keys from the Evidence service and never follows a key URL from +a response. + +## Request one assertion + +```js +execute( + requestEvidence({ + requirement: "urn:example:requirement:adult-status:v1", + purpose: "benefit-eligibility", + audience: "urn:example:agency:benefits", + evidenceType: "urn:example:evidence-type:adult-status:v1", + issuedBy: "urn:example:authority:population-registry", + providedBy: "urn:example:data-service:evidence", + configurationRevision: "sha256:", + expectedAssuranceProfile: "production", + subjects: [ + { + role: "subject", + selectorProfile: "civil-record-v1", + selectorValues: { record_reference: { valueFrom: "person_id" } }, + }, + ], + expectedOutputs: [ + { concept: "urn:example:concept:adult-status", form: "boolean" }, + ], + maximumAssertionLifetimeSeconds: 300, + clockSkewSeconds: 30, + subjectExpectations: { + pinned: [ + { role: "subject", binding: "urn:evidence:subject:v1_" }, + ], + }, + }), +); +``` + +Use `subjectExpectations: "acceptFirstUse"` only for first contact. It verifies +every expectation except prior subject identity, returns +`pinned_subject_expectations`, and requires the workflow to persist and pin +those bindings on later requests. Only pinned expectations prove that a later +assertion concerns the previously accepted subject. + +Selector values referenced with `{ valueFrom: "..." }` are removed from returned +data, including failed operations. For function-resolved inputs, name protected +data paths explicitly in `redactDataPaths`. Configuration remains available to +subsequent operations inside the same OpenFn job. The runtime removes it at the +job output boundary on success and failure. Direct callers must apply an +equivalent boundary before persisting a returned state; do not override the +runtime's default configuration removal. Native clients and prepared requests +remain local variables, never workflow data. + +On success, `state.data.evidence` contains the verified `assertion`, the exact +flattened `jws` bytes for retention, the support correlation `trace_id`, the accepted +subject bindings, and explicit successful verification flags. Use +`selectSupportedValue(assertion, concept)` to select one value by its governed +concept identifier. + +## Progressive request and continuity receipts + +For the SDK's progressive API, mount a reviewed +`registry.evidence-client-profile/v1` JSON profile into the worker and configure: + +```json +{ "evidence": { "profilePath": "/run/evidence/client.json" } } +``` + +The profile owns service expectations, verification policy, metadata trust, +contracts, and a private-key reference. Alternatively supply `privateKeyJwk` +alongside `profilePath` from the OpenFn credential. Profile mode uses the native +`EvidenceClient.fromProfile` API and cannot be combined with explicit SDK config. + +```js +execute(requestEvidence({ + requirement: "adult-status", + selectors: { record_reference: { valueFrom: "person_id" } }, + // Omit bindingReceipt only on first contact. + bindingReceipt: state => state.data.previous_receipt, +})); +``` + +A verified result includes `subject_continuity: { status, receipt }` with status +`firstUse` or `matched`. Persist the opaque receipt and pass it on the next call. +The native SDK validates receipt scope and subject continuity. Profile mode +accepts `requirement`, `selectors` or role-mapped `subjects`, and `bindingReceipt`; +full explicit policy fields belong to the explicit API above. Both modes request +signed JWS only, retain its exact bytes, and publish no native handles. + +## Failure branches + +Typed API and transport failures return one of these workflow branches without +reflecting response details or selector data: + +- `invalid_request` +- `authentication_failed` +- `not_authorized` +- `response_format_not_acceptable` +- `evidence_not_available` +- `retryable_infrastructure` +- `failed` + +Configuration, nonce, signature, trust, contract, policy, and time failures +throw `EvidenceCallerError` and publish no assertion. There is no unsigned +fallback and no hidden retries. Each operation sends at most one Evidence request. +A signed supported value of `false` is successful evidence; +`evidence_not_available` carries no assertion and must never be coerced to false. + +## Verification + +```sh +npm ci --ignore-scripts --no-audit --no-fund +npm run check +``` + +Node.js 22.12.0 or newer is required by the upstream native binding. diff --git a/packages/registry-evidence/configuration-schema.json b/packages/registry-evidence/configuration-schema.json new file mode 100644 index 0000000..0f1a044 --- /dev/null +++ b/packages/registry-evidence/configuration-schema.json @@ -0,0 +1,213 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "evidence_base_url": { + "title": "Registry Evidence URL", + "type": "string", + "description": "Base URL for the Registry Stack Evidence service.", + "format": "uri", + "minLength": 1, + "examples": [ + "https://evidence.example.gov" + ] + }, + "token": { + "title": "Bearer Token", + "type": "string", + "description": "Legacy static bearer token. Use evidence.token for new jobs.", + "writeOnly": true, + "minLength": 1 + }, + "evidence_verification_jwks": { + "title": "Pinned Evidence Verification JWKS", + "type": "string", + "description": "Pinned trusted Evidence verification JWKS JSON. The adaptor never downloads trust material from the assertion or service.", + "minLength": 1 + }, + "evidence_revoked_key_ids": { + "title": "Revoked Evidence Key IDs", + "type": "string", + "description": "JSON array of current service-key RFC 7638 thumbprints. Required even when the reviewed list is empty.", + "minLength": 2, + "default": "[]" + }, + "evidence_trusted_root_certificates": { + "title": "Trusted Root Certificates", + "type": "string", + "description": "Optional PEM certificate bundle for the Evidence service TLS connection.", + "writeOnly": true + }, + "evidence_max_response_bytes": { + "title": "Maximum Signed Response Bytes", + "type": "integer", + "description": "Optional maximum byte size for a signed Evidence response.", + "minimum": 1 + }, + "evidence": { + "title": "Evidence Client Configuration", + "description": "Current SDK configuration, or a mounted progressive client profile. Credentials remain available inside a job and the OpenFn runtime removes configuration at its output boundary.", + "type": "object", + "oneOf": [ + { + "required": [ + "baseUrl", + "trustedJwks", + "revokedKeyIds", + "token" + ], + "not": { + "required": [ + "profilePath" + ] + } + }, + { + "required": [ + "profilePath" + ], + "additionalProperties": false, + "properties": { + "profilePath": { + "type": "string", + "minLength": 1 + }, + "privateKeyJwk": { + "type": "object", + "writeOnly": true + } + } + } + ], + "properties": { + "baseUrl": { + "type": "string", + "format": "uri" + }, + "trustedJwks": { + "type": "object", + "required": [ + "keys" + ], + "properties": { + "keys": { + "type": "array", + "items": { + "type": "object" + } + } + } + }, + "revokedKeyIds": { + "type": "array", + "items": { + "type": "string" + } + }, + "token": { + "type": "object", + "writeOnly": true, + "oneOf": [ + { + "required": [ + "static" + ], + "additionalProperties": false, + "properties": { + "static": { + "type": "string", + "minLength": 1 + } + } + }, + { + "required": [ + "privateKeyJwt" + ], + "additionalProperties": false, + "properties": { + "privateKeyJwt": { + "type": "object", + "required": [ + "tokenEndpoint", + "clientId", + "clientKey" + ], + "properties": { + "tokenEndpoint": { + "type": "string", + "format": "uri" + }, + "clientId": { + "type": "string", + "minLength": 1 + }, + "clientKey": { + "type": "object", + "writeOnly": true + } + } + } + } + } + ] + } + } + } + }, + "oneOf": [ + { + "required": [ + "evidence" + ], + "not": { + "anyOf": [ + { + "required": [ + "evidence_base_url" + ] + }, + { + "required": [ + "token" + ] + }, + { + "required": [ + "evidence_verification_jwks" + ] + }, + { + "required": [ + "evidence_revoked_key_ids" + ] + }, + { + "required": [ + "evidence_trusted_root_certificates" + ] + }, + { + "required": [ + "evidence_max_response_bytes" + ] + } + ] + } + }, + { + "required": [ + "evidence_base_url", + "token", + "evidence_verification_jwks", + "evidence_revoked_key_ids" + ], + "not": { + "required": [ + "evidence" + ] + } + } + ] +} diff --git a/packages/registry-evidence/jobs/request-evidence-profile.js b/packages/registry-evidence/jobs/request-evidence-profile.js new file mode 100644 index 0000000..4ba470f --- /dev/null +++ b/packages/registry-evidence/jobs/request-evidence-profile.js @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: Apache-2.0 +// Mount the reviewed SDK profile and set configuration.evidence.profilePath. +import { execute } from "@openfn/language-common"; +import { requestEvidence } from "../src/index.js"; + +execute(requestEvidence({ + requirement: "adult-status", + selectors: { record_reference: { valueFrom: "person_id" } }, + bindingReceipt: state => state.data.previous_receipt, +})); diff --git a/packages/registry-evidence/jobs/request-evidence.js b/packages/registry-evidence/jobs/request-evidence.js new file mode 100644 index 0000000..88d830e --- /dev/null +++ b/packages/registry-evidence/jobs/request-evidence.js @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// OpenFn workflow template for one signed Registry Stack Evidence request. + +import { execute } from "@openfn/language-common"; +import { requestEvidence } from "../src/index.js"; + +execute( + requestEvidence({ + requirement: "urn:example:requirement:adult-status:v1", + purpose: "benefit-eligibility", + audience: "urn:example:agency:benefits", + evidenceType: "urn:example:evidence-type:adult-status:v1", + issuedBy: "urn:example:authority:population-registry", + providedBy: "urn:example:data-service:evidence", + configurationRevision: `sha256:${"0".repeat(64)}`, + expectedAssuranceProfile: "production", + subjects: [ + { + role: "subject", + selectorProfile: "civil-record-v1", + selectorValues: { record_reference: { valueFrom: "person_id" } }, + }, + ], + expectedOutputs: [ + { concept: "urn:example:concept:adult-status", form: "boolean" }, + ], + maximumAssertionLifetimeSeconds: 300, + clockSkewSeconds: 30, + subjectExpectations: { + pinned: [ + { + role: "subject", + binding: `urn:evidence:subject:v1_${"A".repeat(43)}`, + }, + ], + }, + }), +); diff --git a/packages/registry-evidence/package.json b/packages/registry-evidence/package.json new file mode 100644 index 0000000..167fc89 --- /dev/null +++ b/packages/registry-evidence/package.json @@ -0,0 +1,33 @@ +{ + "name": "@openfn/language-registry-evidence", + "version": "0.1.0", + "description": "OpenFn language helpers for Registry Stack Evidence.", + "license": "Apache-2.0", + "type": "module", + "main": "./src/index.js", + "exports": { + ".": "./src/index.js" + }, + "files": [ + "src", + "jobs", + "configuration-schema.json", + "README.md" + ], + "scripts": { + "test": "node --experimental-vm-modules --no-warnings --test test/evidence.test.js", + "check": "node --check src/index.js && node --check jobs/request-evidence.js && node --check jobs/request-evidence-profile.js && npm test", + "pack:dry-run": "npm pack --dry-run" + }, + "engines": { + "node": ">=22.12.0" + }, + "dependencies": { + "@openfn/language-common": "3.3.4", + "@registrystack/client": "0.27.0" + }, + "devDependencies": { + "@openfn/compiler": "1.3.0", + "@openfn/runtime": "1.9.5" + } +} diff --git a/packages/registry-evidence/src/index.js b/packages/registry-evidence/src/index.js new file mode 100644 index 0000000..f81ba88 --- /dev/null +++ b/packages/registry-evidence/src/index.js @@ -0,0 +1,422 @@ +// SPDX-License-Identifier: Apache-2.0 + +import { createRequire } from "node:module"; +export * from "@openfn/language-common"; + +const require = createRequire(import.meta.url); +const { + EvidenceClient, + EvidenceClientError, +} = require("@registrystack/client").evidence; + +const SPEC_FIELDS = [ + "requirement", + "purpose", + "audience", + "evidenceType", + "issuedBy", + "providedBy", + "configurationRevision", + "expectedAssuranceProfile", + "subjects", + "expectedOutputs", + "maximumAssertionLifetimeSeconds", + "clockSkewSeconds", + "subjectExpectations", +]; +const UNSAFE_PATH_PARTS = new Set(["__proto__", "prototype", "constructor"]); + +export class EvidenceCallerError extends Error { + constructor(message, options = {}) { + super(message); + this.name = "EvidenceCallerError"; + this.code = options.code ?? "evidence_caller.error"; + } +} + +/** Request and verify one signed Registry Stack Evidence assertion. */ +export function requestEvidence(options = {}) { + return async (state) => { + let resolvedOptions = {}; + try { + resolvedOptions = typeof options === "function" ? options(state) : options; + return await callEvidence(state, prepareEvidenceRequest(state, resolvedOptions)); + } catch (error) { + // OpenFn reports the operation's current state on a thrown failure. + // Clear protected inputs there too, without stripping job credentials. + const data = redactedData(state, redactDataPaths(resolvedOptions ?? {})); + delete data[stringOrUndefined(resolvedOptions?.as) ?? "evidence"]; + state.data = data; + delete state.response; + throw callerError(error); + } + }; +} + +/** + * Resolve OpenFn values without network I/O. Explicit mode closes policy and + * generates a nonce; profile mode loads the configured local SDK profile and + * leaves discovery, preparation, and verification to its native request call. + */ +export function prepareEvidenceRequest(state, options = {}) { + if (typeof options === "function") options = options(state); + if (!options || typeof options !== "object" || Array.isArray(options)) { + throw new EvidenceCallerError("request options must be an object", { code: "configuration.request_options" }); + } + const configuration = configurationObject(state); + const clientConfig = evidenceConfiguration(configuration); + if (options.responseFormat !== undefined && options.responseFormat !== "signed-jws") { + throw new EvidenceCallerError("only signed-jws responses are supported", { code: "configuration.response_format" }); + } + const spec = { responseFormat: "signed-jws" }; + for (const field of SPEC_FIELDS) { + if (Object.hasOwn(options, field)) { + spec[field] = resolveInputValue(state, options[field]); + } + } + + try { + const profileMode = clientConfig.profilePath !== undefined; + const unsupported = profileMode + ? SPEC_FIELDS.filter((field) => !["requirement", "subjects"].includes(field)) + : ["selectors", "bindingReceipt"]; + if (unsupported.some((field) => options[field] !== undefined)) { + throw new EvidenceCallerError("request options do not match the configured client mode", { code: "configuration.request_mode" }); + } + const client = profileMode + ? EvidenceClient.fromProfile(clientConfig.profilePath, clientConfig.privateKeyJwk) + : new EvidenceClient(clientConfig); + const progressive = profileMode ? resolveInputValue(state, { + requirement: options.requirement, + responseFormat: "signed-jws", + ...(options.selectors === undefined ? {} : { selectors: options.selectors }), + ...(options.subjects === undefined ? {} : { subjects: options.subjects }), + ...(options.bindingReceipt === undefined ? {} : { bindingReceipt: options.bindingReceipt }), + }) : undefined; + return { + client, + prepared: profileMode ? undefined : client.prepare(spec), + progressive, + as: stringOrUndefined(options.as) ?? "evidence", + requirement: spec.requirement, + purpose: spec.purpose, + redactDataPaths: redactDataPaths(options), + }; + } catch (error) { + throw callerError(error); + } +} + +/** Send one prepared request, retain its exact JWS bytes, and verify offline. */ +export async function callEvidence(state, request) { + if (request.progressive) { + try { + const verified = await request.client.request(request.progressive); + return finish(state, request, { + branch: "succeeded", + trace_id: stringOrUndefined(verified.traceId), + requirement: request.requirement, + assertion: verified.evidence, + jws: verified.assertion.toString("utf8"), + subject_continuity: verified.subjectContinuity, + verification: { authentic: true, currently_valid: true, policy_satisfied: true }, + }); + } catch (error) { + if (error instanceof EvidenceClientError && !["configuration", "nonce", "verification"].includes(error.kind)) { + return finish(state, request, failureResult(error)); + } + throw callerError(error); + } + } + let raw; + try { + raw = await request.client.send(request.prepared); + } catch (error) { + if (error instanceof EvidenceClientError) { + if (["configuration", "nonce", "verification"].includes(error.kind)) { + throw callerError(error); + } + return finish(state, request, failureResult(error)); + } + throw callerError(error); + } + + let verified; + try { + verified = request.client.verify(request.prepared, raw); + } catch (error) { + throw callerError(error); + } + + return finish(state, request, { + branch: "succeeded", + trace_id: stringOrUndefined(raw.traceId) ?? stringOrUndefined(verified.traceId), + requirement: request.requirement, + purpose: request.purpose, + assertion: verified.evidence, + jws: raw.body.toString("utf8"), + pinned_subject_expectations: verified.pinnedSubjectExpectations, + verification: { + authentic: true, + currently_valid: true, + policy_satisfied: true, + }, + }); +} + +/** Select exactly one verified supported value by governed concept. */ +export function selectSupportedValue(assertion, concept) { + const matches = Array.isArray(assertion?.supportedValues) + ? assertion.supportedValues.filter((entry) => entry?.providesValueFor === concept) + : []; + if (matches.length !== 1) { + throw new EvidenceCallerError("expected exactly one supported value for the concept", { + code: "supported_value.cardinality", + }); + } + return matches[0].value; +} + +function failureResult(error) { + const status = error.kind === "not_available" + ? 422 + : (Number.isSafeInteger(error.status) ? error.status : 0); + const retryAfter = Number.isSafeInteger(error.retryAfterSeconds) && error.retryAfterSeconds > 0 + ? error.retryAfterSeconds + : undefined; + let branch = "failed"; + let fallbackCode = "request.failed"; + + if (error.kind === "transport" || (error.kind === "token" && error.tokenKind === "transport")) { + branch = "retryable_infrastructure"; + fallbackCode = "transport.error"; + } else if (error.kind === "not_available") { + branch = "evidence_not_available"; + fallbackCode = "evidence_not_available"; + } else if (error.kind === "token") { + branch = "authentication_failed"; + fallbackCode = "authentication_failed"; + } else if (error.kind === "denied" && status === 401) { + branch = "authentication_failed"; + fallbackCode = "authentication_failed"; + } else if (error.kind === "denied" && status === 403) { + branch = "not_authorized"; + fallbackCode = "not_authorized"; + } else if (error.kind === "denied" && status === 429) { + branch = "retryable_infrastructure"; + fallbackCode = "rate_limited"; + } else if (error.kind === "protocol" && status === 400) { + branch = "invalid_request"; + fallbackCode = "malformed_request"; + } else if (error.kind === "protocol" && status === 406) { + branch = "response_format_not_acceptable"; + fallbackCode = "response_format_not_acceptable"; + } else if (error.kind === "protocol" && status >= 500) { + branch = "retryable_infrastructure"; + fallbackCode = "service_unavailable"; + } + + return { + branch, + trace_id: stringOrUndefined(error.traceId), + ...(retryAfter === undefined ? {} : { retry_after_seconds: retryAfter }), + problem: { + code: safeProblemCode(error.code) ?? fallbackCode, + status, + title: failureTitle(branch), + retryable: branch === "retryable_infrastructure", + }, + }; +} + +function failureTitle(branch) { + switch (branch) { + case "authentication_failed": return "Registry Evidence authentication failed"; + case "not_authorized": return "Registry Evidence request was not authorized"; + case "evidence_not_available": return "Registry Evidence could not be produced"; + case "retryable_infrastructure": return "Registry Evidence is temporarily unavailable"; + case "invalid_request": return "Registry Evidence request was invalid"; + case "response_format_not_acceptable": return "Registry Evidence response format was not acceptable"; + default: return "Registry Evidence request failed"; + } +} + +function callerError(error) { + if (error instanceof EvidenceCallerError) { + return error; + } + if (error instanceof EvidenceClientError) { + const verification = error.kind === "verification"; + return new EvidenceCallerError( + verification + ? "Registry Evidence verification failed" + : "Registry Evidence client configuration was refused", + { code: verification ? `verification.${safeProblemCode(error.code) ?? "failed"}` : `client.${error.kind}` }, + ); + } + return new EvidenceCallerError("Registry Evidence client failed", { code: "client.native_failure" }); +} + +function finish(state, request, result) { + const data = redactedData(state, request.redactDataPaths); + const { response: _response, ...safeState } = state; + return { + ...safeState, + data: { + ...data, + [request.as]: compactObject(result), + }, + }; +} + +function redactedData(state, paths) { + const data = { ...dataObject(state) }; + delete data.evidence_request; + delete data.evidence_context; + for (const path of paths) deleteDataPath(data, path); + return data; +} + +function evidenceConfiguration(configuration) { + const legacyFields = ["evidence_base_url", "token", "evidence_verification_jwks", "evidence_revoked_key_ids", "evidence_trusted_root_certificates", "evidence_max_response_bytes"]; + if (configuration.evidence !== undefined) { + if (legacyFields.some((field) => configuration[field] !== undefined)) { + throw new EvidenceCallerError("configure evidence or legacy Evidence fields, never both", { code: "configuration.authentication" }); + } + const config = parseConfigurationJson(configuration.evidence, "configuration.evidence"); + if (config.profilePath !== undefined && Object.keys(config).some((key) => !["profilePath", "privateKeyJwk"].includes(key))) { + throw new EvidenceCallerError("profilePath cannot be combined with explicit client settings", { code: "configuration.profile" }); + } + return config; + } + const config = { + baseUrl: requireString(configuration.evidence_base_url, "configuration.evidence_base_url"), + trustedJwks: parseConfigurationJson(configuration.evidence_verification_jwks, "configuration.evidence_verification_jwks"), + revokedKeyIds: parseConfigurationJson(configuration.evidence_revoked_key_ids, "configuration.evidence_revoked_key_ids"), + token: { static: requireString(configuration.token, "configuration.token") }, + }; + copyOptionalConfiguration(configuration, config, "evidence_trusted_root_certificates", "trustedRootCertificates"); + copyOptionalConfiguration(configuration, config, "evidence_max_response_bytes", "maxResponseBytes"); + return config; +} + +function parseConfigurationJson(value, label) { + if (value && typeof value === "object") { + return value; + } + const text = requireString(value, label); + try { + return JSON.parse(text); + } catch (_error) { + throw new EvidenceCallerError(`${label} must contain valid JSON`, { code: "configuration.invalid_json" }); + } +} + +function copyOptionalConfiguration(source, target, sourceKey, targetKey) { + const value = source[sourceKey]; + if (value !== undefined && value !== null && value !== "") { + target[targetKey] = value; + } +} + +function resolveInputValue(state, value) { + if (typeof value === "function") { + return value(state); + } + if (Array.isArray(value)) { + return value.map((item) => resolveInputValue(state, item)); + } + if (value && typeof value === "object") { + if (typeof value.valueFrom === "string") { + return valueFromPath(dataObject(state), value.valueFrom); + } + const resolved = {}; + for (const [key, item] of Object.entries(value)) { + resolved[key] = resolveInputValue(state, item); + } + return resolved; + } + return value; +} + +function valueFromPath(data, path) { + const parts = safePathParts(path); + if (!parts || parts.length === 0) return undefined; + let current = data; + for (const part of parts) { + if (!current || typeof current !== "object" || Array.isArray(current) || !Object.hasOwn(current, part)) { + return undefined; + } + current = current[part]; + } + return current; +} + +function redactDataPaths(options) { + const paths = new Set(Array.isArray(options.redactDataPaths) ? options.redactDataPaths : []); + collectValueFromPaths(options, paths); + return [...paths].filter((path) => safePathParts(path)?.length > 0); +} + +function collectValueFromPaths(value, paths) { + if (Array.isArray(value)) { + for (const item of value) collectValueFromPaths(item, paths); + } else if (value && typeof value === "object") { + if (typeof value.valueFrom === "string") paths.add(value.valueFrom); + for (const item of Object.values(value)) collectValueFromPaths(item, paths); + } +} + +function deleteDataPath(data, path) { + const parts = safePathParts(path); + if (!parts || parts.length === 0) return; + if (parts.length === 1) { + delete data[parts[0]]; + return; + } + let current = data; + for (let index = 0; index < parts.length - 1; index += 1) { + const part = parts[index]; + if (!current || typeof current !== "object" || Array.isArray(current) || !Object.hasOwn(current, part)) return; + const next = current[part]; + if (!next || typeof next !== "object" || Array.isArray(next)) return; + current[part] = { ...next }; + current = current[part]; + } + delete current[parts.at(-1)]; +} + +function safePathParts(path) { + if (typeof path !== "string" || path.length === 0) return undefined; + const parts = path.split(".").filter(Boolean); + return parts.some((part) => UNSAFE_PATH_PARTS.has(part)) ? undefined : parts; +} + +function safeProblemCode(value) { + return typeof value === "string" && /^[a-z0-9][a-z0-9._-]{0,127}$/.test(value) ? value : undefined; +} + +function compactObject(value) { + return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined)); +} + +function dataObject(state) { + return state?.data && typeof state.data === "object" && !Array.isArray(state.data) ? state.data : {}; +} + +function configurationObject(state) { + return state?.configuration && typeof state.configuration === "object" && !Array.isArray(state.configuration) + ? state.configuration + : {}; +} + +function requireString(value, label) { + if (typeof value !== "string" || value.length === 0) { + throw new EvidenceCallerError(`${label} is required`, { code: "configuration.required" }); + } + return value; +} + +function stringOrUndefined(value) { + return typeof value === "string" && value.length > 0 ? value : undefined; +} diff --git a/packages/registry-evidence/test/evidence.test.js b/packages/registry-evidence/test/evidence.test.js new file mode 100644 index 0000000..cc12ae7 --- /dev/null +++ b/packages/registry-evidence/test/evidence.test.js @@ -0,0 +1,616 @@ +import assert from "node:assert/strict"; +import { createHash, generateKeyPairSync, randomBytes, sign, verify } from "node:crypto"; +import { existsSync, readFileSync, mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import http from "node:http"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +import compile from "@openfn/compiler"; +import run from "@openfn/runtime"; +import { + EvidenceCallerError, + fn, + execute, + prepareEvidenceRequest, + requestEvidence, + selectSupportedValue, +} from "../src/index.js"; + +const CONFIGURATION_REVISION = `sha256:${"0".repeat(64)}`; +const SUBJECT_BINDING = `urn:evidence:subject:v1_${randomBytes(32).toString("base64url")}`; +const TEMPLATE_SUBJECT_BINDING = `urn:evidence:subject:v1_${"A".repeat(43)}`; +const TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736"; +const TRACEPARENT = `00-${TRACE_ID}-00f067aa0ba902b7-01`; +const packageRoot = fileURLToPath(new URL("..", import.meta.url)); +const languageCommonRoot = findDependencyRoot(packageRoot, "@openfn/language-common"); + +test("adaptor reexports common operations used by Lightning autoimports", () => { + assert.equal(typeof fn, "function"); + assert.equal(typeof execute, "function"); +}); + +test("prepareEvidenceRequest uses the native client to close policy with fresh nonces", () => { + const signingKey = generateSigningKey(); + const state = evidenceState("http://127.0.0.1:1/", signingKey.jwks); + const first = prepareEvidenceRequest(state, requestOptions()); + const second = prepareEvidenceRequest(state, requestOptions()); + + assert.match(first.prepared.requestNonce, /^[A-Za-z0-9_-]{43}$/); + assert.notEqual(first.prepared.requestNonce, second.prepared.requestNonce); + assert.equal(first.prepared.policyDocument.requestNonce, first.prepared.requestNonce); + assert.equal(first.prepared.policyDocument.audience, "urn:example:agency:benefits"); + assert.deepEqual(first.prepared.subjectExpectations, "acceptFirstUse"); + assert.deepEqual(first.redactDataPaths, ["person_id"]); +}); + +test("requestEvidence sends and verifies through the real Registry Stack client", async () => { + const signingKey = generateSigningKey(); + const spec = resolvedRequestSpec(); + const stub = await startStubServer({ + "POST /v1/evidence": evidenceRoute(spec, signingKey), + }); + + try { + const state = await requestEvidence(requestOptions())( + evidenceState(stub.baseUrl, signingKey.jwks), + ); + + assert.equal(stub.requests.length, 1); + assert.equal(stub.requests[0].headers.authorization, "Bearer secret-token"); + assert.equal(stub.requests[0].headers.accept, "application/jose+json"); + assert.deepEqual(JSON.parse(stub.requests[0].body), { + requestNonce: state.data.evidence.assertion.requestNonce, + requirement: spec.requirement, + purpose: spec.purpose, + subjects: [{ + role: "subject", + selector: { + profile: "civil-record-v1", + values: { record_reference: "person-123" }, + }, + }], + }); + assert.equal(state.data.evidence.branch, "succeeded"); + assert.equal(state.data.evidence.trace_id, TRACE_ID); + assert.equal(state.data.evidence.verification.authentic, true); + assert.equal(state.data.evidence.verification.currently_valid, true); + assert.equal(state.data.evidence.verification.policy_satisfied, true); + assert.equal( + selectSupportedValue(state.data.evidence.assertion, "urn:example:concept:adult-status"), + true, + ); + assert.deepEqual(state.data.evidence.pinned_subject_expectations, [ + { role: "subject", binding: SUBJECT_BINDING }, + ]); + assert.equal(typeof state.data.evidence.jws, "string"); + assert.equal("person_id" in state.data, false); + assert.equal(state.configuration.token, "secret-token"); + assert.equal(JSON.stringify(state.data).includes("secret-token"), false); + assert.equal(JSON.stringify(state).includes("person-123"), false); + } finally { + await stub.close(); + } +}); + +test("native verification rejects a signed response outside the closed policy", async () => { + const signingKey = generateSigningKey(); + const spec = resolvedRequestSpec(); + const stub = await startStubServer({ + "POST /v1/evidence": evidenceRoute(spec, signingKey, { + audience: "urn:example:agency:other", + }), + }); + + try { + await assert.rejects( + () => requestEvidence(requestOptions())(evidenceState(stub.baseUrl, signingKey.jwks)), + (error) => error instanceof EvidenceCallerError + && error.code.startsWith("verification."), + ); + } finally { + await stub.close(); + } +}); + +for (const [scenario, expectedCode] of [ + ["wrong purpose", "verification.policy"], + ["expired assertion", "verification.time"], + ["tampered signature", "verification.signature"], +]) { + test(`native verification refuses ${scenario} without releasing result or response residue`, async () => { + const signingKey = generateSigningKey(); + const spec = resolvedRequestSpec(); + const instant = Date.now(); + const timestamp = offset => new Date(instant + offset).toISOString().replace(/\.\d+Z$/, "Z"); + const overrides = scenario === "wrong purpose" + ? { purpose: "different-governed-purpose" } + : scenario === "expired assertion" + ? { issuedAt: timestamp(-180_000), observedAt: timestamp(-180_000), validUntil: timestamp(-120_000) } + : {}; + let originalJws; + let responseJws; + const stub = await startStubServer({ + "POST /v1/evidence": (_req, res, body) => { + const assertion = evidenceFor(spec, JSON.parse(body).requestNonce, overrides); + originalJws = JSON.parse(signEvidence(assertion, signingKey)); + responseJws = { ...originalJws }; + if (scenario === "tampered signature") { + const signature = Buffer.from(responseJws.signature, "base64url"); + signature[0] ^= 1; + responseJws.signature = signature.toString("base64url"); + } + res.writeHead(200, { "content-type": "application/jose+json", traceparent: TRACEPARENT }); + res.end(JSON.stringify(responseJws)); + }, + }); + try { + const state = evidenceState(stub.baseUrl, signingKey.jwks); + const configuration = state.configuration; + state.response = { body: "untrusted-response-residue" }; + state.data.evidence = { assertion: { residue: "previous-assertion" }, jws: "previous-jws" }; + let published; + await assert.rejects(async () => { + published = await requestEvidence(requestOptions())(state); + }, error => error instanceof EvidenceCallerError && error.code === expectedCode); + assert.equal(published, undefined); + assert.equal(stub.requests.length, 1); + assert.equal(state.configuration, configuration); + assert.equal(state.configuration.token, "secret-token"); + assert.equal(state.data.evidence, undefined); + assert.equal(state.response, undefined); + assert.equal(state.data.person_id, undefined); + for (const residue of ["previous-assertion", "previous-jws", "untrusted-response-residue", "person-123"]) { + assert.equal(JSON.stringify(state.data).includes(residue), false); + } + // Distinguish a correctly signed policy/time refusal from malformed JWS. + const checkSignature = jws => verify("sha256", Buffer.from(`${jws.protected}.${jws.payload}`), + { key: signingKey.privateKey, dsaEncoding: "ieee-p1363" }, Buffer.from(jws.signature, "base64url")); + assert.equal(checkSignature(originalJws), true); + assert.equal(Buffer.from(responseJws.signature, "base64url").length, 64); + assert.equal(checkSignature(responseJws), scenario !== "tampered signature"); + if (scenario === "expired assertion") { + const assertion = JSON.parse(Buffer.from(responseJws.payload, "base64url")); + assert.equal(Date.parse(assertion.validUntil) - Date.parse(assertion.issuedAt), 60_000); + assert.ok(Date.parse(assertion.validUntil) + spec.clockSkewSeconds * 1000 < instant); + } + } finally { await stub.close(); } + }); +} + +test("native revocation checking rejects a still-trusted signing key", async () => { + const signingKey = generateSigningKey(); + const spec = resolvedRequestSpec(); + const stub = await startStubServer({ + "POST /v1/evidence": evidenceRoute(spec, signingKey), + }); + + try { + const state = evidenceState(stub.baseUrl, signingKey.jwks, [signingKey.kid]); + await assert.rejects( + () => requestEvidence(requestOptions())(state), + (error) => error instanceof EvidenceCallerError + && error.code.startsWith("verification."), + ); + } finally { + await stub.close(); + } +}); + +test("typed Evidence failures map to redacted workflow branches", async () => { + const signingKey = generateSigningKey(); + const unavailable = await startStubServer({ + "POST /v1/evidence": (_req, res) => { + res.writeHead(422, { + "content-type": "application/problem+json", + traceparent: TRACEPARENT, + }); + res.end(problemBody(422, "evidence.unavailable")); + }, + }); + + try { + const state = await requestEvidence(requestOptions())( + evidenceState(unavailable.baseUrl, signingKey.jwks), + ); + assert.deepEqual(state.data.evidence.problem, { + code: "evidence_not_available", + status: 422, + title: "Registry Evidence could not be produced", + retryable: false, + }); + assert.equal(state.data.evidence.branch, "evidence_not_available"); + assert.equal(JSON.stringify(state).includes("protected person-123"), false); + assert.equal(JSON.stringify(state).includes("person-123"), false); + } finally { + await unavailable.close(); + } +}); + +test("429 failures preserve the typed retry interval", async () => { + const signingKey = generateSigningKey(); + const stub = await startStubServer({ + "POST /v1/evidence": (_req, res) => { + res.writeHead(429, { + "content-type": "application/problem+json", + "retry-after": "30", + }); + res.end(problemBody(429, "evidence.rate_limited")); + }, + }); + + try { + const state = await requestEvidence(requestOptions())( + evidenceState(stub.baseUrl, signingKey.jwks), + ); + assert.equal(state.data.evidence.branch, "retryable_infrastructure"); + assert.equal(state.data.evidence.retry_after_seconds, 30); + assert.equal(state.data.evidence.problem.retryable, true); + } finally { + await stub.close(); + } +}); + +test("template imports the Evidence helper and avoids generic HTTP helpers", () => { + const template = readFileSync(new URL("../jobs/request-evidence.js", import.meta.url), "utf8"); + assert.match(template, /requestEvidence/); + assert.doesNotMatch(template, /@openfn\/language-http/); + assert.doesNotMatch(template, /Authorization/); +}); + +test("compiled OpenFn template executes the current native client", async () => { + const signingKey = generateSigningKey(); + const spec = resolvedRequestSpec({ expectedAssuranceProfile: "production" }); + const stub = await startStubServer({ + "POST /v1/evidence": evidenceRoute(spec, signingKey, { + subjects: [{ role: "subject", binding: TEMPLATE_SUBJECT_BINDING }], + }), + }); + const template = readFileSync(new URL("../jobs/request-evidence.js", import.meta.url), "utf8"); + const { code } = compile(template); + + try { + const result = await run( + { + workflow: { + steps: [{ id: "request-evidence", expression: code }], + start: "request-evidence", + }, + options: { start: "request-evidence" }, + }, + evidenceState(stub.baseUrl, signingKey.jwks), + { + linker: { + modules: { + "@openfn/language-common": { path: languageCommonRoot }, + "../src/index.js": { path: packageRoot }, + }, + cacheKey: `openfn-evidence-native-test-${process.pid}-${Date.now()}`, + }, + }, + ); + + assert.equal(result.errors, undefined); + assert.equal(stub.requests.length, 1); + assert.equal(result.data.evidence.branch, "succeeded"); + assert.equal(result.data.evidence.assertion.supportedValues[0].value, true); + assert.equal("person_id" in result.data, false); + assert.equal("configuration" in result, false); + } finally { + await stub.close(); + } +}); + +test("nested SDK configuration exchanges privateKeyJwt once without leaking credentials", async () => { + const signingKey = generateSigningKey(); + const clientKey = generateSigningKey(); + const clientJwk = { ...clientKey.privateKey.export({ format: "jwk" }), kid: clientKey.kid, alg: "ES256" }; + const stub = await startStubServer({ + "POST /token": (_req, res, body) => { + const form = new URLSearchParams(body); + assert.equal(form.get("grant_type"), "client_credentials"); + assert.equal(form.get("client_assertion_type"), "urn:ietf:params:oauth:client-assertion-type:jwt-bearer"); + const [header, payload, signature] = form.get("client_assertion").split("."); + assert.equal(JSON.parse(Buffer.from(payload, "base64url")).sub, "openfn-pilot"); + assert.equal(verify("sha256", Buffer.from(`${header}.${payload}`), { key: clientKey.privateKey, dsaEncoding: "ieee-p1363" }, Buffer.from(signature, "base64url")), true); + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ access_token: "minted-token", token_type: "Bearer", expires_in: 300 })); + }, + "POST /v1/evidence": evidenceRoute(resolvedRequestSpec(), signingKey), + }); + try { + const input = evidenceState(stub.baseUrl, signingKey.jwks); + input.configuration = { evidence: { + baseUrl: stub.baseUrl, trustedJwks: signingKey.jwks, revokedKeyIds: [], + token: { privateKeyJwt: { tokenEndpoint: `${stub.baseUrl}token`, clientId: "openfn-pilot", clientKey: clientJwk } }, + } }; + const result = await requestEvidence(requestOptions())(input); + assert.equal(result.data.evidence.branch, "succeeded"); + assert.equal(stub.requests.length, 2); + assert.equal(stub.requests[1].headers.authorization, "Bearer minted-token"); + assert.equal(JSON.stringify(result.data).includes(clientJwk.d), false); + assert.equal(JSON.stringify(result.data).includes("minted-token"), false); + } finally { await stub.close(); } +}); + +test("signed false is available evidence and pinned continuity refuses a different subject", async () => { + const signingKey = generateSigningKey(); + const stub = await startStubServer({ "POST /v1/evidence": evidenceRoute(resolvedRequestSpec(), signingKey, { + supportedValues: [{ providesValueFor: "urn:example:concept:adult-status", value: false }], + }) }); + try { + const result = await requestEvidence(requestOptions())(evidenceState(stub.baseUrl, signingKey.jwks)); + assert.equal(result.data.evidence.branch, "succeeded"); + assert.equal(selectSupportedValue(result.data.evidence.assertion, "urn:example:concept:adult-status"), false); + await assert.rejects(requestEvidence(requestOptions({ subjectExpectations: { pinned: [{ role: "subject", binding: TEMPLATE_SUBJECT_BINDING }] } }))(evidenceState(stub.baseUrl, signingKey.jwks)), { name: "EvidenceCallerError" }); + assert.equal(stub.requests.length, 2); + } finally { await stub.close(); } +}); + +test("runtime retains credentials between operations and strips them on success and verification failure", async () => { + for (const invalidSignature of [false, true]) { + const signingKey = generateSigningKey(); + const responseKey = invalidSignature ? generateSigningKey() : signingKey; + const stub = await startStubServer({ "POST /v1/evidence": evidenceRoute(resolvedRequestSpec(), responseKey) }); + try { + const spec = JSON.stringify({ ...resolvedRequestSpec(), redactDataPaths: ["person_id"] }); + const { code } = compile(` + import { execute } from "@openfn/language-common"; + import { requestEvidence } from "../src/index.js"; + execute( + requestEvidence(state => (${spec})), + state => { if (state.configuration.token !== "secret-token") throw new Error("configuration lost"); return state; }, + requestEvidence(state => (${spec})) + ); + `); + const result = await run({ workflow: { steps: [{ id: "evidence", expression: code }], start: "evidence" }, options: { start: "evidence" } }, evidenceState(stub.baseUrl, signingKey.jwks), { + linker: { modules: { "@openfn/language-common": { path: languageCommonRoot }, "../src/index.js": { path: packageRoot } }, cacheKey: `runtime-boundary-${invalidSignature}-${Date.now()}` }, + }); + assert.equal("configuration" in result, false); + assert.equal(JSON.stringify(result).includes("secret-token"), false); + assert.equal(JSON.stringify(result).includes("person-123"), false); + assert.equal(stub.requests.length, invalidSignature ? 1 : 2); + if (invalidSignature) { + assert.ok(result.errors); + assert.equal(result.data.evidence, undefined); + } else { + assert.equal(result.errors, undefined); + assert.equal(result.data.evidence.branch, "succeeded"); + assert.notEqual(JSON.parse(stub.requests[0].body).requestNonce, JSON.parse(stub.requests[1].body).requestNonce); + } + } finally { await stub.close(); } + } +}); + +test("ambiguous credential sources and unsigned response requests are refused locally", () => { + const key = generateSigningKey(); + const state = evidenceState("http://127.0.0.1:1/", key.jwks); + state.configuration.evidence = { baseUrl: "http://127.0.0.1:1/" }; + assert.throws(() => prepareEvidenceRequest(state, requestOptions()), { code: "configuration.authentication" }); + delete state.configuration.evidence; + assert.throws(() => prepareEvidenceRequest(state, requestOptions({ responseFormat: "sd-jwt-vc" })), { code: "configuration.response_format" }); +}); + +test("profile requests verify first-use and matched receipts and refuse subject changes", async () => { + const signingKey = generateSigningKey(); + const clientKey = generateSigningKey(); + const spec = resolvedRequestSpec(); + const directory = mkdtempSync(resolve(tmpdir(), "openfn-evidence-profile-")); + let binding = SUBJECT_BINDING; + let origin; + const jsonRoute = (value, contentType = "application/json") => (_req, res) => { + res.writeHead(200, { "content-type": contentType }); + res.end(JSON.stringify(typeof value === "function" ? value() : value)); + }; + const stub = await startStubServer({ + "GET /.well-known/oauth-protected-resource": jsonRoute(() => ({ resource: origin, authorization_servers: [origin], jwks_uri: `${origin}/.well-known/evidence/jwks.json`, bearer_methods_supported: ["header"] })), + "GET /.well-known/oauth-authorization-server": jsonRoute(() => ({ issuer: origin, token_endpoint: `${origin}/token`, grant_types_supported: ["client_credentials"], token_endpoint_auth_methods_supported: ["private_key_jwt"] })), + "GET /.well-known/evidence/jwks.json": jsonRoute(signingKey.jwks, "application/jwk-set+json"), + "POST /token": jsonRoute({ access_token: "profile-token", token_type: "Bearer", expires_in: 300 }), + "GET /v1/evidence-definitions": jsonRoute({ + schema: "registry.evidence-definitions/v1", assuranceProfile: "local", audience: spec.audience, issuedBy: spec.issuedBy, providedBy: spec.providedBy, + definitions: [{ handle: "adult-status", requirement: spec.requirement, configurationRevision: spec.configurationRevision, kind: "criterion", evidenceType: spec.evidenceType, purpose: spec.purpose, responseFormats: ["signed-jws"], referenceFrameworks: ["urn:example:framework:adult-status"], subjects: [{ role: "subject", cardinality: "one", selector: { profile: "civil-record-v1", valueOrigin: "request", fields: [{ type: "string", name: "record_reference", minimumBytes: 1, maximumBytes: 200 }] } }], concepts: [{ handle: "adult", concept: "urn:example:concept:adult-status", required: true, form: "boolean" }] }], + }), + "POST /v1/evidence": (req, res, body) => evidenceRoute(spec, signingKey, { subjects: [{ role: "subject", binding }] })(req, res, body), + }); + origin = stub.baseUrl.replace(/\/$/, ""); + try { + const profilePath = resolve(directory, "client.json"); + writeFileSync(profilePath, JSON.stringify({ schema: "registry.evidence-client-profile/v1", baseUrl: origin, clientId: "openfn-profile", privateKey: { source: "file", path: "unused-in-memory-key.jwk" }, trust: { type: "local-loopback-discovery" }, contracts: { type: "published" } }), { mode: 0o600 }); + const configuration = { evidence: { profilePath, privateKeyJwk: { ...clientKey.privateKey.export({ format: "jwk" }), kid: clientKey.kid, alg: "ES256" } } }; + const options = { requirement: "adult-status", selectors: { record_reference: { valueFrom: "person_id" } } }; + const input = () => ({ configuration, data: { person_id: "person-123" } }); + const first = await requestEvidence(options)(input()); + assert.equal(first.data.evidence.subject_continuity.status, "firstUse"); + const receipt = first.data.evidence.subject_continuity.receipt; + assert.equal(JSON.stringify(receipt).includes("person-123"), false); + const second = await requestEvidence({ ...options, bindingReceipt: receipt })(input()); + assert.equal(second.data.evidence.subject_continuity.status, "matched"); + assert.equal(second.data.evidence.trace_id, TRACE_ID); + assert.equal(typeof second.data.evidence.jws, "string"); + binding = TEMPLATE_SUBJECT_BINDING; + await assert.rejects(requestEvidence({ ...options, bindingReceipt: receipt })(input()), { name: "EvidenceCallerError" }); + assert.equal(stub.requests.filter(request => request.url === "/v1/evidence").length, 3); + } finally { await stub.close(); rmSync(directory, { recursive: true, force: true }); } +}); + +function requestOptions(overrides = {}) { + return { + requirement: "urn:example:requirement:adult-status:v1", + purpose: "benefit-eligibility", + audience: "urn:example:agency:benefits", + evidenceType: "urn:example:evidence-type:adult-status:v1", + issuedBy: "urn:example:authority:population-registry", + providedBy: "urn:example:data-service:evidence", + configurationRevision: CONFIGURATION_REVISION, + expectedAssuranceProfile: "local", + subjects: [{ + role: "subject", + selectorProfile: "civil-record-v1", + selectorValues: { record_reference: { valueFrom: "person_id" } }, + }], + expectedOutputs: [ + { concept: "urn:example:concept:adult-status", form: "boolean" }, + ], + maximumAssertionLifetimeSeconds: 300, + clockSkewSeconds: 30, + subjectExpectations: "acceptFirstUse", + ...overrides, + }; +} + +function resolvedRequestSpec(overrides = {}) { + const options = requestOptions(overrides); + return { + ...options, + subjects: [{ + role: "subject", + selectorProfile: "civil-record-v1", + selectorValues: { record_reference: "person-123" }, + }], + }; +} + +function evidenceState(baseUrl, jwks, revokedKeyIds = []) { + return { + data: { + request_id: "workflow-request-1", + person_id: "person-123", + }, + configuration: { + evidence_base_url: baseUrl, + token: "secret-token", + evidence_verification_jwks: JSON.stringify(jwks), + evidence_revoked_key_ids: JSON.stringify(revokedKeyIds), + }, + }; +} + +function generateSigningKey() { + const { publicKey, privateKey } = generateKeyPairSync("ec", { namedCurve: "prime256v1" }); + const jwk = publicKey.export({ format: "jwk" }); + const thumbprint = JSON.stringify({ crv: jwk.crv, kty: jwk.kty, x: jwk.x, y: jwk.y }); + const kid = createHash("sha256").update(thumbprint).digest("base64url"); + return { + kid, + privateKey, + jwks: { keys: [{ ...jwk, kid, alg: "ES256" }] }, + }; +} + +function evidenceRoute(spec, signingKey, overrides = {}) { + return (_req, res, body) => { + const requestBody = JSON.parse(body); + const evidence = evidenceFor(spec, requestBody.requestNonce, overrides); + res.writeHead(200, { + "content-type": "application/jose+json", + traceparent: TRACEPARENT, + }); + res.end(signEvidence(evidence, signingKey)); + }; +} + +function evidenceFor(spec, nonce, overrides = {}) { + const issuedAt = new Date().toISOString().replace(/\.\d+Z$/, "Z"); + const validUntil = new Date(Date.now() + 60_000).toISOString().replace(/\.\d+Z$/, "Z"); + return { + schema: "registry.assertion-evidence/v1", + assuranceProfile: spec.expectedAssuranceProfile, + subjectBinding: "audience-scoped", + requestNonce: nonce, + id: "urn:example:openfn-test:evidence:1", + type: "Evidence", + supportsRequirement: spec.requirement, + isConformantTo: spec.evidenceType, + issuedBy: spec.issuedBy, + providedBy: spec.providedBy, + issuedAt, + observedAt: issuedAt, + validUntil, + purpose: spec.purpose, + audience: spec.audience, + configurationRevision: spec.configurationRevision, + subjects: [{ role: "subject", binding: SUBJECT_BINDING }], + supportedValues: [{ + providesValueFor: "urn:example:concept:adult-status", + value: true, + }], + ...overrides, + }; +} + +function signEvidence(evidence, signingKey) { + const protectedSegment = Buffer.from(JSON.stringify({ + alg: "ES256", + kid: signingKey.kid, + typ: "evidence+jws", + cty: "application/evidence+json", + })).toString("base64url"); + const payloadSegment = Buffer.from(JSON.stringify(evidence)).toString("base64url"); + const signingInput = `${protectedSegment}.${payloadSegment}`; + const signature = sign("sha256", Buffer.from(signingInput), { + key: signingKey.privateKey, + dsaEncoding: "ieee-p1363", + }); + return JSON.stringify({ + protected: protectedSegment, + payload: payloadSegment, + signature: signature.toString("base64url"), + }); +} + +function problemBody(status, code) { + const [title, detail] = { + "evidence.unavailable": ["Evidence could not be produced", "evidence could not be produced for this request"], + "evidence.rate_limited": ["Evidence request rate is exhausted", "the Evidence request rate is exhausted"], + }[code]; + return JSON.stringify({ + type: `https://id.registrystack.org/problems/registry-evidence/${code.replaceAll(".", "/")}`, + title, status, detail, code, traceId: TRACE_ID, + }); +} + +function startStubServer(routes) { + const requests = []; + const table = new Map(Object.entries(routes)); + const server = http.createServer((req, res) => { + const chunks = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => { + const body = Buffer.concat(chunks).toString("utf8"); + requests.push({ method: req.method, url: req.url, headers: req.headers, body }); + const handler = table.get(`${req.method} ${req.url}`); + if (!handler) { + res.writeHead(404, { "content-type": "text/plain" }); + res.end("no route"); + return; + } + res.setHeader("traceparent", TRACEPARENT); + handler(req, res, body); + }); + }); + + return new Promise((resolvePromise, rejectPromise) => { + server.once("error", rejectPromise); + server.listen(0, "127.0.0.1", () => { + const { port } = server.address(); + resolvePromise({ + baseUrl: `http://127.0.0.1:${port}/`, + requests, + close: () => new Promise((done) => server.close(done)), + }); + }); + }); +} + +function findDependencyRoot(start, packageName) { + let current = resolve(start); + while (true) { + const candidate = resolve(current, "node_modules", packageName); + if (existsSync(resolve(candidate, "package.json"))) return candidate; + const parent = resolve(current, ".."); + if (parent === current) throw new Error(`dependency not found: ${packageName}`); + current = parent; + } +} diff --git a/pilot/agriculture/.gitignore b/pilot/agriculture/.gitignore new file mode 100644 index 0000000..1da3bdd --- /dev/null +++ b/pilot/agriculture/.gitignore @@ -0,0 +1,2 @@ +.runtime/ +__pycache__/ diff --git a/pilot/agriculture/README.md b/pilot/agriculture/README.md new file mode 100644 index 0000000..2fe0f55 --- /dev/null +++ b/pilot/agriculture/README.md @@ -0,0 +1,89 @@ +# Synthetic agricultural holding pilot + +This project adapts the maintained Registry Stack agricultural-holdings core +starter. It retains its Farm, holder, responsibility and reviewed name-correction +model, including the original PublicSchema attribution. All supplied names, +identifiers, inputs and identities are synthetic. + +The complete deployment and OpenFn workflows are described in the repository +README. `prepare.py --bin-dir ` creates an ignored, +owner-only `.runtime` directory. It needs Python 3 with PyYAML and OpenSSL. +Preparation generates fresh keys, fixed service bindings, separate database roles, +a direct BREG source export, a compiled Evidence bundle and a native client +profile. It executes eight Evidence fixtures before producing the deployment. +It does not connect to a database or start services. Existing output is refused; +normal restart reuses it and retains records and credentials. + +`initialize.py` runs inside the deployment tools container after the pilot Mint +and PostgreSQL are ready. It uses `bregctl test`, `package`, `apply --initial`, and +`verify`. The schema test database is separate from the pilot record database. +It never resets an existing database or clears a migration interlock. An +interrupted activation needs the maintained BREG recovery procedure. + +## Authority and disclosure + +- `openfn-service` creates farms, resolves their unique `localIdentifier`, and + creates/submits its own corrections. It has no review, application, direct farm + patch, collection listing, or holder maintenance authority. +- The separate `reviewer` principal can inspect, approve and explicitly apply a + proposed correction. `excludeSubmitter: true` and manual application remain + authored BREG requirements. OpenFn receives no reviewer private key. +- `evidence-source` can only perform an exact Farm lookup returning + `localIdentifier`. It cannot read names, list records or write. +- `holding-registered` signs one Boolean concept. It means a unique holding is + currently resolved in this synthetic register. It makes no claim about subsidy + eligibility, land rights, activity, ownership or an individual's identity. + A missing holding produces `evidence.unavailable`, never signed `false`. +- Committed Farm create/patch events project only `local-identifier`. A correction + proposal or approval does not produce a Farm patch event. Delivery uses the + existing transactional outbox and HMAC contract. + +The inherited reader/editor/reviewer registry-wide collection and history grants +remain intentional learning profiles. Production deployments must review their +population scope. The two integration profiles deliberately permit any exact +synthetic Farm identifier, so caller filters are never treated as authorization. + +## Explicit reviewer steps + +Copy the correction request UUID from the registration/correction workflow result. +Run these commands through the deployment's tools service, which has the private +reviewer credential. `inspect` prints the synthetic before/after proposal and saves +that exact response. An approval or application uses its advertised action, +proposal version, effect digest and lifecycle ETag. Stale snapshots are refused. + +```sh +deployment/compose.sh run --rm tools \ + python3 /opt/pilot/agriculture/review.py inspect REQUEST_UUID +deployment/compose.sh run --rm tools \ + python3 /opt/pilot/agriculture/review.py approve REQUEST_UUID +# Approval leaves the authoritative Farm unchanged. Inspect the approved request. +deployment/compose.sh run --rm tools \ + python3 /opt/pilot/agriculture/review.py inspect REQUEST_UUID +deployment/compose.sh run --rm tools \ + python3 /opt/pilot/agriculture/review.py apply REQUEST_UUID +``` + +The supplied `inputs/registration.json` and `inputs/correction.json` use stable +submission identifiers for replay-safe workflow keys. Change both the submission +identifier and local holding identifier for another synthetic registration. +Repeating a correction with changed content requires a new submission identifier. + +## Native verification + +The retained starter journey checks typed holder relationships and dates, and the +reviewed correction lifecycle. The integration journey exercises the new service +creation and both authorized exact lookups. The real HTTP smoke separately proves +refusal of direct farm patch and Evidence-source creation, source name exclusion, +reviewer approval without mutation, explicit application, a cryptographically +verified registered assertion, and missing-record unavailability: + +```sh +deployment/compose.sh run --rm tools \ + python3 /opt/pilot/agriculture/smoke.py +``` + +The smoke uses `SYNTHETIC-DIRECT-SMOKE-001`, separate from the workflow input farm. +It retains that synthetic record and privately stores request verification contexts +under `.runtime/breg/smoke`. Repeated smoke runs create fresh synthetic correction +proposals. They do not reset a database. Its committed events can reach the pilot +workflow once the event bridge is enabled. diff --git a/pilot/agriculture/evidence-starter/derivations/holding-registered.rhai b/pilot/agriculture/evidence-starter/derivations/holding-registered.rhai new file mode 100644 index 0000000..f1c4d41 --- /dev/null +++ b/pilot/agriculture/evidence-starter/derivations/holding-registered.rhai @@ -0,0 +1,4 @@ +fn answer(facts, selectors, context) { + let identifier = required(facts["local-identifier"], "identifier_missing"); + #{ registered: true } +} diff --git a/pilot/agriculture/evidence-starter/fixtures/holding-registered.yaml b/pilot/agriculture/evidence-starter/fixtures/holding-registered.yaml new file mode 100644 index 0000000..da92a56 --- /dev/null +++ b/pilot/agriculture/evidence-starter/fixtures/holding-registered.yaml @@ -0,0 +1,160 @@ +{ + "fixture": "registry.evidence.reference.holding-registered/v1", + "synthetic_only": true, + "common": { + "observed_at": "2026-09-06T00:00:00Z", + "selectors": { + "subject": { + "profile": "breg-8-registry-4-farm-19-by-local-identifier", + "values": { + "local-identifier": "SYNTHETIC-FARM-001" + } + } + }, + "expectedRequestParts": { + "query": [ + { + "name": "accessProfile", + "value": "evidence-source" + }, + { + "name": "$select", + "value": "localIdentifier" + } + ], + "body": { + "selector": "by-local-identifier", + "values": { + "localIdentifier": "SYNTHETIC-FARM-001" + } + } + }, + "expectedTransport": { + "path": "/v1/records/farms:lookup", + "fixedHeaders": [ + { + "name": "Accept", + "value": "application/json" + } + ] + } + }, + "cases": [ + { + "id": "positive", + "response": { + "data": { + "domainData": { + "localIdentifier": "SYNTHETIC-FARM-001", + "name": "SYNTHETIC-PRIVATE-NAME" + } + } + }, + "expected": { + "lookup": "match", + "derivationRuns": true, + "signed": true, + "facts": { + "local-identifier": "SYNTHETIC-FARM-001" + }, + "value": true + } + }, + { + "id": "negative-missing-holding", + "declaredUnresolved": true, + "expected": { + "publicProblem": "evidence.unavailable", + "derivationRuns": false, + "signed": false, + "sourceRequestCount": 1 + } + }, + { + "id": "boundary-wrong-identity", + "response": { + "data": { + "domainData": { + "localIdentifier": "SYNTHETIC-WRONG" + } + } + }, + "expected": { + "error": "source_protocol_error", + "publicProblem": "source.unavailable", + "derivationRuns": false, + "signed": false + } + }, + { + "id": "missing-identity", + "response": { + "data": { + "domainData": {} + } + }, + "expected": { + "error": "source_protocol_error", + "publicProblem": "source.unavailable", + "derivationRuns": false, + "signed": false + } + }, + { + "id": "wrong-identity-type", + "response": { + "data": { + "domainData": { + "localIdentifier": 7 + } + } + }, + "expected": { + "error": "source_protocol_error", + "publicProblem": "source.unavailable", + "derivationRuns": false, + "signed": false + } + }, + { + "id": "source-failure", + "sourceFailure": "timeout", + "expected": { + "publicProblem": "source.unavailable", + "signed": false + } + }, + { + "id": "output-gate-refuses-protected-reference", + "derivationMutation": "return-raw-reference", + "expected": { + "outputGate": "rejected", + "signed": false + } + }, + { + "id": "anti-reconstruction", + "bundleMutation": "duplicate-disclosure-family", + "expected": { + "bundle": "rejected" + } + } + ], + "privacyExpectation": { + "evidenceContains": [ + "urn:example:concept:holding-registered:registered" + ], + "evidenceExcludes": [ + "localIdentifier", + "SYNTHETIC-FARM-001", + "SYNTHETIC-WRONG", + "SYNTHETIC-PRIVATE-NAME", + "breg-8-registry-4-farm-19-by-local-identifier" + ], + "diagnosticsExclude": [ + "SYNTHETIC-FARM-001", + "SYNTHETIC-WRONG", + "SYNTHETIC-PRIVATE-NAME" + ] + } +} diff --git a/pilot/agriculture/evidence-starter/questions/holding-registered.yaml b/pilot/agriculture/evidence-starter/questions/holding-registered.yaml new file mode 100644 index 0000000..d0136cf --- /dev/null +++ b/pilot/agriculture/evidence-starter/questions/holding-registered.yaml @@ -0,0 +1,41 @@ +{ + "id": "holding-registered", + "question": "Is this agricultural holding registered?", + "purpose": "holding-verification", + "subject": { + "role": "subject", + "profiles": [ + "breg-8-registry-4-farm-19-by-local-identifier" + ] + }, + "source": { + "ref": "holding-register" + }, + "answers": [ + { + "concept": "registered", + "id": "urn:example:concept:holding-registered:registered", + "type": "boolean" + } + ], + "derivation": "derivations/holding-registered.rhai", + "disclosure": { + "allow": [ + "registered" + ] + }, + "governance": { + "requirement": "urn:example:requirement:holding-registered:v1", + "kind": "criterion", + "referenceFrameworks": [ + "urn:example:framework:holding-register:v1" + ], + "evidenceType": "urn:example:evidence:holding-registered", + "validitySeconds": 300, + "observationTimezone": "UTC", + "fixtures": "fixtures/holding-registered.yaml", + "disclosureFamilies": [ + "urn:example:disclosure-family:holding-registered" + ] + } +} diff --git a/pilot/agriculture/evidence-starter/targets/local/settings.yaml b/pilot/agriculture/evidence-starter/targets/local/settings.yaml new file mode 100644 index 0000000..570c6aa --- /dev/null +++ b/pilot/agriculture/evidence-starter/targets/local/settings.yaml @@ -0,0 +1,135 @@ +{ + "formatVersion": 1, + "governance": { + "version": 1, + "assuranceProfile": "local", + "service": { + "providerId": "urn:example:provider:record-evidence", + "trustDomain": "urn:example:trust:record-evidence", + "publicOrigin": "http://127.0.0.1:8080" + }, + "issuer": { + "id": "urn:example:issuer:record-evidence" + }, + "authentication": { + "kind": "oidc-access-token", + "issuer": "http://127.0.0.1:8091", + "audiences": [ + "urn:example:audience:agriculture-pilot" + ], + "tokenTypes": [ + "at+jwt" + ], + "algorithms": [ + "ES256" + ], + "jwksUri": "http://127.0.0.1:8091/.well-known/jwks.json", + "principalClaim": "sub", + "requesterTagsClaim": "evidence_tags", + "evidenceAudienceClaim": "evidence_audience", + "grantIdClaim": "evidence_grant_id", + "grantAuthorityClaim": "evidence_authority", + "maximumTokenLifetimeSeconds": 300, + "revokedKeyIds": [] + }, + "audit": { + "format": "keyed-jsonl", + "hashSecretRef": "secret:file/audit-hmac-key", + "hashKeyVersion": 1, + "failClosed": true + }, + "subjectBinding": { + "secretRef": "secret:file/subject-binding-hmac-key", + "keyVersion": 1 + }, + "rateLimits": { + "requestsPerPrincipalPerMinute": 60, + "burstPerPrincipal": 10, + "failedSelectorAttemptsPerPrincipalAuthorityPerMinute": 10 + }, + "signing": { + "format": "flattened-jws-json", + "algorithm": "ES256", + "publishedPublicJwkFiles": [], + "revokedKeyIds": [], + "jwksPath": "/.well-known/evidence/jwks.json", + "maximumAssertionValiditySeconds": 300, + "verifierClockSkewSeconds": 30 + }, + "responseFormats": [ + "signed-jws" + ], + "sourceConnections": { + "registry": { + "baseUrl": "http://127.0.0.1:8090", + "authentication": { + "kind": "oauth2-client-credentials", + "tokenEndpoint": "http://127.0.0.1:8091/token", + "clientIdRef": "secret:file/registry-client-id", + "clientAssertionKeyRef": "secret:file/registry-client-key", + "maximumCacheSeconds": 60 + }, + "concurrencyLimit": 4, + "admissionTimeoutMilliseconds": 5000, + "tokenTimeoutMilliseconds": 5000 + } + }, + "authorityProfiles": { + "holding-verifier": { + "kind": "explicit-request", + "requesterTags": [ + "holding-verifier" + ], + "grants": [ + { + "requirement": "urn:example:requirement:holding-registered:v1", + "purpose": "holding-verification", + "audienceFrom": "authenticated-requester", + "responseFormats": [ + "signed-jws" + ], + "subjects": [ + { + "role": "subject", + "selectorProfile": "breg-8-registry-4-farm-19-by-local-identifier", + "valueOrigin": "request" + } + ] + } + ] + } + } + }, + "runtime": { + "version": 1, + "bundleDirectory": "/absolute/path/to/candidate/bundle", + "listener": { + "bindHost": "127.0.0.1", + "port": 8080, + "tlsTermination": "operator-controlled-upstream", + "trustProxyIdentityHeaders": false, + "maximumRequestBytes": 65536, + "maximumConcurrentRequests": 64, + "requestTimeoutMilliseconds": 10000, + "shutdownGraceMilliseconds": 30000 + }, + "secretProviders": { + "file": { + "root": "/absolute/path/to/evidence/secrets" + } + }, + "signer": { + "kind": "local-jwk", + "privateKeyRef": "secret:file/signing-p256-private-jwk" + }, + "auditStorage": { + "path": "/absolute/path/to/evidence/audit/evidence.jsonl", + "maximumFileBytes": 1073741824 + }, + "outboundTls": { + "systemRoots": true, + "trustProfiles": {} + } + }, + "publicKeys": {} +} diff --git a/pilot/agriculture/initialize.py b/pilot/agriculture/initialize.py new file mode 100644 index 0000000..bb2b7dc --- /dev/null +++ b/pilot/agriculture/initialize.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Rehearse, package, and activate only the prepared synthetic Compose database. + +Run inside the tools container after Mint and the new BREG database are ready. +A repeat verifies the retained package and does not seed or reset records. +""" +import json +import os +from pathlib import Path +import subprocess +import sys + +ROOT = Path('/config/breg') + + +def run(binary, *args): + result = subprocess.run([binary, *map(str,args)], capture_output=True, text=True) + if result.returncode: + # Product diagnostics exclude tokens and record values. Never print stdout, + # since mint's success output contains an access token. + detail = result.stderr if binary == 'mint' else result.stderr + result.stdout + raise RuntimeError(f'{binary} {args[0]} failed: {detail[-5000:]}') + return result.stdout + + +def save(path, value): + temporary = path.with_name(path.name+'.tmp') + temporary.write_text(value if isinstance(value,str) else json.dumps(value,indent=2)+'\n') + temporary.chmod(0o600) + temporary.replace(path) + + +def main(): + os.umask(0o077) + os.environ['SSL_CERT_FILE']=str(ROOT/'secrets/database-ca.pem') + runtime=ROOT/'runtime.json' + if (ROOT/'initialized.json').exists(): + run('bregctl','verify','--runtime-config',runtime) + print('Retained agriculture package verified; records and credentials preserved.') + return + if not (ROOT/'schema-test-receipt.json').exists(): + clients=json.loads((ROOT/'registry/clients.yaml').read_text())['clients'] + for client in clients: + token=run('mint','token','--url','http://127.0.0.1:8091/token','--client-id',client['id'],'--key',ROOT/f"clients/{client['id']}/signing-p256-private-jwk").strip() + if token.count('.') != 2 or any(c.isspace() for c in token): + raise RuntimeError('Mint returned an invalid token shape') + for profile in client['accessProfiles']: + save(ROOT/f'secrets/token-{profile}',token) + report=run('bregctl','--format','json','test',ROOT/'registry','--runtime-config',ROOT/'test-runtime.json','--credentials',ROOT/'schema-test-credentials.json','--database-id','agriculture-pilot-db','--output',ROOT/'schema-test-receipt.json') + save(ROOT/'test-report.json',report) + if not (ROOT/'build/package').exists(): + if (ROOT/'build').exists(): + raise RuntimeError('Incomplete package build exists; inspect it before retrying. No files were removed.') + report=run('bregctl','--format','json','package',ROOT/'registry','--database-id','agriculture-pilot-db','--test-receipt',ROOT/'schema-test-receipt.json','--output',ROOT/'build') + save(ROOT/'package-report.json',report) + revision=json.loads((ROOT/'package-report.json').read_text())['packageRevision'] + configuration=json.loads((ROOT/'runtime-template.json').read_text()) + configuration['package']['activeRevision']=revision + save(runtime,configuration) + # --initial is the maintained first activation flow. Never clear an interlock + # or mutate the database with an improvised recovery command. + run('bregctl','apply','--runtime-config',runtime,'--package',ROOT/'build/package','--initial') + run('bregctl','verify','--runtime-config',runtime) + save(ROOT/'initialized.json',{'packageRevision':revision}) + print('Agriculture schema journeys passed and the exact local package was activated.') + + +if __name__=='__main__': + try: + main() + except Exception as error: + print(f'Initialization stopped: {error}',file=sys.stderr) + sys.exit(1) diff --git a/pilot/agriculture/inputs/correction.json b/pilot/agriculture/inputs/correction.json new file mode 100644 index 0000000..7aec0f9 --- /dev/null +++ b/pilot/agriculture/inputs/correction.json @@ -0,0 +1,7 @@ +{ + "submissionId": "synthetic-correction-001", + "localIdentifier": "SYNTHETIC-FARM-001", + "name": "Synthetic River Holding", + "reason": "Synthetic spelling correction", + "supportingReference": "SYNTHETIC-SUPPORT-001" +} diff --git a/pilot/agriculture/inputs/evidence.json b/pilot/agriculture/inputs/evidence.json new file mode 100644 index 0000000..538eb4a --- /dev/null +++ b/pilot/agriculture/inputs/evidence.json @@ -0,0 +1,3 @@ +{ + "localIdentifier": "SYNTHETIC-FARM-001" +} diff --git a/pilot/agriculture/inputs/registration.json b/pilot/agriculture/inputs/registration.json new file mode 100644 index 0000000..50fefa7 --- /dev/null +++ b/pilot/agriculture/inputs/registration.json @@ -0,0 +1,5 @@ +{ + "submissionId": "synthetic-registration-001", + "localIdentifier": "SYNTHETIC-FARM-001", + "name": "Synthetci River Holding" +} diff --git a/pilot/agriculture/prepare.py b/pilot/agriculture/prepare.py new file mode 100644 index 0000000..cdc8164 --- /dev/null +++ b/pilot/agriculture/prepare.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Prepare a new, private synthetic deployment. Never connects to a database.""" +import argparse +import json +import os +from pathlib import Path +import secrets +import shutil +import subprocess +import yaml + +HERE = Path(__file__).resolve().parent +PROFILE = 'breg-8-registry-4-farm-19-by-local-identifier' +REQUIREMENT = 'urn:example:requirement:holding-registered:v1' + + +def write(path, value): + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + with path.open('x', encoding='utf-8') as handle: + handle.write(value if isinstance(value, str) else json.dumps(value, indent=2) + '\n') + path.chmod(0o600) + + +def run(binary, *args): + result = subprocess.run([str(binary), *map(str, args)], capture_output=True, text=True) + if result.returncode: + raise RuntimeError(f'{Path(binary).name} {args[0]} failed: {result.stderr[-3000:]}') + return result.stdout + + +def prepare(root, bins): + if root.exists(): + raise SystemExit('Output exists. Reuse the retained runtime, or choose a new --output directory.') + for name in ['bregctl','evidencectl','evidence','mint']: + if run(bins/name,'--version').split()[-1] != '0.27.0': + raise SystemExit('This pilot requires matching Registry Stack 0.27.0 binaries.') + os.umask(0o077) + root.mkdir(parents=True, mode=0o700) + bregctl, evidencectl = bins/'bregctl', bins/'evidencectl' + os.environ['EVIDENCE_BIN'] = str(bins/'evidence') + for name in ['breg', 'mint', 'bridge', 'evidence', 'evidence-client', 'openfn', 'breg-db']: + (root/name).mkdir(mode=0o700) + breg = root/'breg' + shutil.copytree(HERE/'registry', breg/'registry') + clients = json.loads((HERE/'registry/clients.yaml').read_text())['clients'] + mint_keys = root/'mint/keys' + run(evidencectl, 'keygen', 'signing', '--output-dir', mint_keys) + mint_public = json.loads((mint_keys/'signing-p256-public.jwk.json').read_text()) + mint_kid = mint_public['kid'] + run(evidencectl, 'keygen', 'secret', '--output', mint_keys/'audit-hmac-key') + write(root/f'mint/public-keys/{mint_kid}.jwk.json', mint_public) + for client in clients + [{'id':'openfn-evidence','scopes':['evidence:request'],'claims':{'evidence_tags':['holding-verifier'],'evidence_audience':'urn:example:audience:openfn-pilot'}}]: + ident = client['id'] + key_dir = breg/f'clients/{ident}' if ident != 'openfn-evidence' else root/'evidence-client/keys' + run(evidencectl, 'keygen', 'signing', '--output-dir', key_dir) + public = json.loads((key_dir/'signing-p256-public.jwk.json').read_text()) + write(root/f'mint/clients/{ident}.yaml', {'clientId':ident,'principal':f'urn:example:pilot:{ident}','authorization':{'scopes':client['scopes'],'claims':client['claims']},'keys':[public]}) + write(root/'mint/mint.json', {'version':1,'validationMode':'supervised-local-development','issuer':'http://127.0.0.1:8091','listener':{'address':'127.0.0.1','port':8091},'signing':{'algorithm':'ES256','activePublicJwkFile':f'public-keys/{mint_kid}.jwk.json','publishedPublicJwkFiles':[],'revokedKeyIds':[]},'signer':{'kind':'local-jwk','privateKeyRef':'secret:file/signing-p256-private-jwk'},'secretProviders':{'file':{'root':'/config/mint/keys'}},'audit':{'path':'/var/lib/mint/mint.jsonl','maximumFileBytes':10485760,'hashKeyRef':'secret:file/audit-hmac-key','hashKeyVersion':1},'accessTokens':{'audiences':['urn:example:audience:agriculture-pilot'],'lifetimeSeconds':300},'clientAssertion':{'audience':'http://127.0.0.1:8091/token','maximumLifetimeSeconds':120,'algorithms':['ES256']},'clients':{'directory':'clients'}}) + for name in ['audit-key','cursor-key','webhook-key']: + write(breg/f'secrets/{name}', secrets.token_hex(32)) + write(breg/'secrets/mint-jwks', {'keys':[mint_public]}) + password = secrets.token_hex(24) + write(root/'breg-db/password', password) + write(root/'breg-db/postgres.env', f'POSTGRES_USER=postgres\nPOSTGRES_PASSWORD={password}\nPOSTGRES_DB=postgres\n') + for mode,database in [('', 'agriculture'),('test-', 'agriculture_test')]: + for role in ['runtime','migration']: + write(breg/f'secrets/{mode}{role}-database-url', f'postgresql://agriculture_{role}:{password}@breg-db:5432/{database}') + sql = ''.join(f"CREATE ROLE agriculture_{role} LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOBYPASSRLS PASSWORD '{password}';\n" for role in ['runtime','migration']) + for db in ['agriculture', 'agriculture_test']: + sql += f'CREATE DATABASE {db};\n\\connect {db}\nCREATE EXTENSION IF NOT EXISTS btree_gist;\nREVOKE ALL ON DATABASE {db} FROM PUBLIC;\nGRANT CONNECT ON DATABASE {db} TO agriculture_runtime, agriculture_migration;\n' + for schema in ['registry_internal','registry_data','registry_source','registry_derived','registry_context']: + sql += f'CREATE SCHEMA {schema} AUTHORIZATION agriculture_migration;\nREVOKE ALL ON SCHEMA {schema} FROM PUBLIC;\n' + write(root/'breg-db/bootstrap.sql', sql) + run('openssl','req','-x509','-new','-nodes','-newkey','rsa:2048','-sha256','-days','30','-subj','/CN=Synthetic agriculture pilot CA','-keyout',root/'breg-db/ca.key','-out',root/'breg-db/ca.pem') + run('openssl','req','-new','-nodes','-newkey','rsa:2048','-subj','/CN=breg-db','-keyout',root/'breg-db/server.key','-out',root/'breg-db/server.csr') + write(root/'breg-db/server.ext','subjectAltName=DNS:breg-db\n') + run('openssl','x509','-req','-sha256','-days','30','-in',root/'breg-db/server.csr','-CA',root/'breg-db/ca.pem','-CAkey',root/'breg-db/ca.key','-CAcreateserial','-extfile',root/'breg-db/server.ext','-out',root/'breg-db/server.crt') + shutil.copyfile(root/'breg-db/ca.pem',breg/'secrets/database-ca.pem') + runtime = {'apiVersion':'registry.registrystack.org/breg-runtime/v1alpha1','kind':'BRegRuntimeConfig','listener':{'bind':'0.0.0.0:8090'},'identity':{'environment':'local','instanceId':'agriculture-openfn-pilot','databaseId':'agriculture-pilot-db','databaseInitializationEnvironment':'local'},'secretProviders':{'file':{'root':'/config/breg/secrets'}},'database':{'runtimeUrlRef':'secret:file/runtime-database-url','migrationUrlRef':'secret:file/migration-database-url','pool':{'maxSize':4},'roles':{'migration':'agriculture_migration','runtime':'agriculture_runtime'}},'package':{'root':'/config/breg/build/package','trustAnchorPath':'/config/breg/trust-anchor.json','compilerSourceRevision':'agriculture-openfn-pilot-v1','activeRevision':'sha256:'+'1'*64,'activeSequence':1},'authentication':{'oidc':{'issuer':'http://127.0.0.1:8091','audience':'urn:example:audience:agriculture-pilot','allowedAlgorithm':'ES256','accessTokenType':'at+jwt','scopeClaim':'scope','scopeSeparator':' ','allowedClients':[c['id'] for c in clients],'deniedKids':[],'maxTokenLifetimeSeconds':300,'leewayMilliseconds':30000,'jwksSource':{'kind':'static','documentRef':'secret:file/mint-jwks'}},'authorityClaims':{'principal':'registry_principal','purpose':'registry_purpose'}},'audit':{'hashKeyRef':'secret:file/audit-key'},'cursor':{'secretRef':'secret:file/cursor-key'},'eventDestinations':{'openfn':{'origin':'http://127.0.0.1:8081','path':'/events/breg','networkProfile':'loopbackDevelopmentHttp','dnsFamily':'dualStackStrict','allowedPrivateCidrs':[],'hmacSha256KeyRef':'secret:file/webhook-key','classificationCeiling':'restricted','deliveryCeilings':{'attemptTimeoutMilliseconds':5000,'maximumAttempts':5}}},'eventDelivery':{'payloadRetentionDays':1}} + write(breg/'runtime-template.json', runtime) + test = json.loads(json.dumps(runtime)); test['database'].update(runtimeUrlRef='secret:file/test-runtime-database-url',migrationUrlRef='secret:file/test-migration-database-url'); test['package']['root']='/config/breg/empty-package' + (breg/'empty-package').mkdir() + write(breg/'test-runtime.json',test); write(breg/'trust-anchor.json',{}) + bindings=[] + journeys=json.loads((breg/'registry/tests/journeys.yaml').read_text()) + for journey in journeys['journeys']: + for step in journey['steps']: + bindings.append({'journeyId':journey['id'],'stepId':step['id'],'credential':{'type':'bearer','tokenRef':'secret:file/token-'+step['accessProfile']}}) + write(breg/'schema-test-credentials.json',{'apiVersion':'registry.registrystack.org/breg-schema-test-credentials/v1','kind':'SchemaTestCredentials','bindings':bindings}) + write(root/'bridge/breg-hmac-key',(breg/'secrets/webhook-key').read_text()) + events={} + for event in ['farm-created-v1','farm-patched-v1']: + sample=json.loads(run(bregctl,'webhook','sample',breg/'registry','--event',event,'--format','json')) + events[event]={'schema':sample['request']['headers']['ce-dataschema'],'trigger':sample['request']['body']['trigger']} + write(root/'bridge/expected-events.json',events); write(root/'bridge/allowed-value-fields.json',['local-identifier']) + write(root/'bridge/expected-source','urn:registrystack:registry:agricultural-holdings:instance:agriculture-openfn-pilot') + evidence=root/'evidence'; project=evidence/'project' + run(evidencectl,'new',project,'--starter',HERE/'evidence-starter','--profile','local') + settings=json.loads((HERE/'evidence-starter/targets/local/settings.yaml').read_text()) + settings['runtime']['bundleDirectory']=str(evidence/'candidate/bundle'); settings['runtime']['secretProviders']['file']['root']=str(project/'secrets'); settings['runtime']['auditStorage']['path']='/var/lib/evidence/evidence.jsonl' + write(evidence/'settings.json',settings) + run(evidencectl,'target','new',evidence/'target','--settings',evidence/'settings.json','--signing-public-key',project/'secrets/signing-p256-public.jwk.json') + run(bregctl,'generate','evidence-source',breg/'registry','--access-profile','evidence-source','--entity','farm','--selector','by-local-identifier','--fields','local-identifier','--source-id','holding-register','--connection','registry','--output',evidence/'source-export') + run(evidencectl,'source','import',evidence/'source-export','--project',project,'--target',evidence/'target') + fixture_report=run(evidencectl,'fixtures','run','--project',project,'--target',evidence/'target','--json'); write(evidence/'fixture-report.json',fixture_report) + run(evidencectl,'build','--project',project,'--target',evidence/'target','--output',evidence/'candidate') + runtime_path=evidence/'candidate/runtime.yaml'; ev_runtime=yaml.safe_load(runtime_path.read_text()); ev_runtime['bundleDirectory']='/config/evidence/candidate/bundle'; ev_runtime['secretProviders']['file']['root']='/config/evidence/project/secrets'; write(evidence/'runtime.yaml',ev_runtime) + (evidence/'runtime.yaml').chmod(0o400) + write(project/'secrets/registry-client-id','evidence-source') + write(project/'secrets/registry-client-key',(breg/'clients/evidence-source/signing-p256-private-jwk').read_text()) + run(evidencectl,'client','profile','create','--base-url','http://127.0.0.1:8080','--client-id','openfn-evidence','--private-key-file','keys/signing-p256-private-jwk','--local-loopback-discovery','--expected-audience','urn:example:audience:openfn-pilot','--output',root/'evidence-client/profile.json') + key=json.loads((breg/'clients/openfn-service/signing-p256-private-jwk').read_text()) + breg_credential={'breg':{'baseUrl':'http://127.0.0.1:8090','authorization':{'privateKeyJwt':{'tokenEndpoint':'http://127.0.0.1:8091/token','clientId':'openfn-service','clientKey':key}}}} + write(root/'openfn/breg-credential.json',breg_credential) + write(root/'openfn/evidence-credential.json',{'evidence':{'profilePath':'/config/evidence-client/profile.json'}}) + write(root/'workflow-bindings.json',{'breg':{'accessProfile':'openfn-service','farmEntity':'farm','farmCreateOperation':'records.farm.create','selector':'by-local-identifier','correctionEntity':'name-correction','correctionCreateOperation':'records.name-correction.create','correctionSubmitOperation':'records.name-correction.request.submit'},'evidence':{'requirement':'holding-registered','requirementId':REQUIREMENT,'selectorProfile':PROFILE,'purpose':'holding-verification','registeredConcept':'urn:example:concept:holding-registered:registered'}}) + write(root/'prepared.json',{'schema':'synthetic-agriculture-pilot/v1','version':'0.27.0','fixtures':8}) + print('Prepared isolated synthetic configuration, source export and 8 passing Evidence fixtures.') + + +if __name__ == '__main__': + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--bin-dir',type=Path,required=True) + parser.add_argument('--output',type=Path,default=HERE/'.runtime') + args=parser.parse_args() + prepare(args.output.resolve(),args.bin_dir.resolve()) diff --git a/pilot/agriculture/registry/ATTRIBUTION.md b/pilot/agriculture/registry/ATTRIBUTION.md new file mode 100644 index 0000000..14038f7 --- /dev/null +++ b/pilot/agriculture/registry/ATTRIBUTION.md @@ -0,0 +1,16 @@ +# PublicSchema attribution + +This starter adapts selected concepts from [PublicSchema](https://publicschema.org/), +by the PublicSchema contributors, licensed under +[Creative Commons Attribution 4.0 International](https://creativecommons.org/licenses/by/4.0/). + +The exact draft source is commit +[`1ea9ce333918693b29aec31068fac412e02cb8dc`](https://github.com/PublicSchema/publicschema.org/tree/1ea9ce333918693b29aec31068fac412e02cb8dc). +The version string is 0.3.0; the commit, rather than that unchanged version +string, identifies the draft vocabulary used here. The distribution includes +`PUBLICSCHEMA-LICENSE.txt`. + +Registry Stack adapted the vocabulary into an operational profile: minimal +identity fields, native typed references, fixed permissions, date constraints, +reviewed corrections and synthetic examples. Those choices are documented in +`MODEL.md`. No endorsement or certification by PublicSchema is implied. diff --git a/pilot/agriculture/registry/MODEL.md b/pilot/agriculture/registry/MODEL.md new file mode 100644 index 0000000..519d131 --- /dev/null +++ b/pilot/agriculture/registry/MODEL.md @@ -0,0 +1,81 @@ +# Fixed model and policy + +This is a synthetic local learning profile. All access is authenticated and +restricted to the declared profiles. Browser previews never authorize local +requests. The registry-wide grants and historical reads are explicit policy: +there are no hidden jurisdiction or ownership row filters. An institution must +review these boundaries before production deployment. + +## Actors and operations + +| Actor | Ordinary records | Correction requests | +| --- | --- | --- | +| Reader | Get and list Farm records only; no person, organization, informal-group or holder-role access | No access | +| Editor | Create, get, list and inspect history; PATCH only where the entity is not controlled | Create and edit own drafts; submit, revise or cancel own requests | +| Reviewer | Get, list and inspect history | Read proposed fields and target fields, approve, reject, request revision and apply | + +Every operation absent from the authored grants is denied, including deletion. +The full PATCH operation on Farm is controlled. The correction changes +only `name`; it cannot change identity or holder/link endpoints. A reviewer is +also the authorized applier. Review excludes the submitting principal, even +if that person acquires a reviewer role. Approval leaves the target unchanged. +Application uses the native frozen effect and expected target revision. A stale +target requires the native revision/rebase and review path. + +Reasons and supporting references live in correction records, restricted to +maintainers. They are bounded synthetic text, not uploaded evidence or proof of +legal authority. Historical access includes old values of readable fields. + +## References, identity and dates + +BReg assigns the record UUID. `localIdentifier` is a unique, bounded local key +within each concrete entity, not a national identifier. It is this profile's +minimal representation of an Identifier, not the complete PublicSchema +multivalued `identifiers` structure. Class concept IRIs describe types; they are +never used as instance IDs. Importers must map external instance IRIs to the +correct concrete entity and returned UUID before writing references. Name +matching is not identity resolution. + +Every endpoint is a native reference to one declared entity, with restricted +reference deletion. Missing and wrong-type targets are refused. No independently +editable inverse list duplicates the forward endpoints. All supplied dates are +native calendar dates. Both-known date pairs enforce start <= end, including +an equal-date assertion. Missing dates remain unknown; dates do not establish +currentness, exclusivity or non-overlap. Examples use the fixed date 2026-01-01. + +The sample identifiers start `SYNTHETIC-`; `MY-FIRST-001` belongs only to the +independent first-record exercise. That exercise requires no sample references. +Edit `examples/inputs/first-record.json` before its first run. Native retained +state binds exact input bytes and captured UUIDs to each attempt. + +## Semantic scope + +Farm is a production unit, not a Group. Person, Organization and InformalGroup +are separate minimal identities. The three concrete holder roles target Farm +and exactly Person, Organization or InformalGroup. HoldingOperatorRole is +abstract and has no route. The draft permits broader Group types, while this +fixed profile deliberately restricts group holders to InformalGroup. + +A holder bears major resource decisions and management control. Joint holders +are represented by separate assertions. Holders are not automatically workers +or daily managers, and registration gives no title, entitlement or civil identity. +A holding needs no land area, parcel, national ID, bank account or household. +The portable Manifest projection describes the model from the editor profile; +it grants no data access. Caller-filtered runtime metadata and reads still obey +each selected profile. Only editor and reviewer profiles may read identity and +holder-role records. +The reader profile sees production-unit names and local identifiers only. +The editor may directly create and PATCH persons, organizations, informal groups +and holder roles. These operations are not implicitly governed by Farm review. +The samples show joint individual, cooperative and informal-group holders, +including a landless holding. Parcel, worker and farmer-registration profiles +are outside this package. + +## Attribution and adoption + +The model maps selected fields and native references to PublicSchema draft +concepts at commit `1ea9ce333918693b29aec31068fac412e02cb8dc`, under CC BY 4.0. +See `PUBLICSCHEMA-LICENSE.txt` and the pinned source in `starter-template.json`. +The local key representation, permissions, review policy, reference target +narrowing and validation rules are Registry Stack starter decisions. Local +institutional and domain review remains necessary before adopting this profile. diff --git a/pilot/agriculture/registry/PUBLICSCHEMA-LICENSE.txt b/pilot/agriculture/registry/PUBLICSCHEMA-LICENSE.txt new file mode 100644 index 0000000..fa4e7d3 --- /dev/null +++ b/pilot/agriculture/registry/PUBLICSCHEMA-LICENSE.txt @@ -0,0 +1,40 @@ +Creative Commons Attribution 4.0 International + +This license applies to the reference model in this repository: +concept definitions, property definitions, vocabulary value sets, and +credential schemas (everything under the schema/ directory). + +For the full license text, see: +https://creativecommons.org/licenses/by/4.0/legalcode + +You are free to: + + Share - copy and redistribute the material in any medium or format + for any purpose, even commercially. + + Adapt - remix, transform, and build upon the material for any purpose, + even commercially. + +Under the following terms: + + Attribution - You must give appropriate credit, provide a link to the + license, and indicate if changes were made. You may do so in any + reasonable manner, but not in any way that suggests the licensor + endorses you or your use. + + No additional restrictions - You may not apply legal terms or + technological measures that legally restrict others from doing + anything the license permits. + +Copyright 2026 PublicSchema Contributors + +## Attribution Guidance + +Embedding the PublicSchema JSON-LD context URL +(https://publicschema.org/ctx/draft.jsonld) in a credential, dataset, +or application counts as sufficient attribution under CC-BY-4.0. + +No additional notice is required in end-user-facing interfaces. If you +redistribute modified definitions from the reference model, include a +link to the original at https://publicschema.org and a note indicating +changes were made. diff --git a/pilot/agriculture/registry/clients.yaml b/pilot/agriculture/registry/clients.yaml new file mode 100644 index 0000000..be59602 --- /dev/null +++ b/pilot/agriculture/registry/clients.yaml @@ -0,0 +1,70 @@ +{ + "version": 1, + "clients": [ + { + "id": "reader", + "accessProfiles": [ + "reader" + ], + "scopes": [ + "starter:reader" + ], + "claims": { + "registry_principal": "synthetic-reader", + "registry_purpose": "starter-learning" + } + }, + { + "id": "editor", + "accessProfiles": [ + "editor" + ], + "scopes": [ + "starter:editor" + ], + "claims": { + "registry_principal": "synthetic-editor", + "registry_purpose": "starter-learning" + } + }, + { + "id": "reviewer", + "accessProfiles": [ + "reviewer" + ], + "scopes": [ + "starter:reviewer" + ], + "claims": { + "registry_principal": "synthetic-reviewer", + "registry_purpose": "starter-learning" + } + }, + { + "id": "openfn-service", + "accessProfiles": [ + "openfn-service" + ], + "scopes": [ + "pilot:submit" + ], + "claims": { + "registry_principal": "synthetic-openfn-service", + "registry_purpose": "agriculture-registration" + } + }, + { + "id": "evidence-source", + "accessProfiles": [ + "evidence-source" + ], + "scopes": [ + "pilot:evidence-source" + ], + "claims": { + "registry_principal": "synthetic-evidence-source", + "registry_purpose": "evidence-source-read" + } + } + ] +} diff --git a/pilot/agriculture/registry/examples/inputs/first-record.json b/pilot/agriculture/registry/examples/inputs/first-record.json new file mode 100644 index 0000000..38291ca --- /dev/null +++ b/pilot/agriculture/registry/examples/inputs/first-record.json @@ -0,0 +1,6 @@ +{ + "record": { + "localIdentifier": "MY-FIRST-001", + "name": "My Frist Holding" + } +} diff --git a/pilot/agriculture/registry/examples/inputs/reviewed-change.json b/pilot/agriculture/registry/examples/inputs/reviewed-change.json new file mode 100644 index 0000000..7b568a0 --- /dev/null +++ b/pilot/agriculture/registry/examples/inputs/reviewed-change.json @@ -0,0 +1,10 @@ +{ + "change": { + "record": { + "recordRef": "first-record" + }, + "name": "My First Holding", + "reason": "Correct a synthetic spelling mistake.", + "supportingReference": "SYNTHETIC-REVIEW-001" + } +} diff --git a/pilot/agriculture/registry/examples/inputs/starter-data.json b/pilot/agriculture/registry/examples/inputs/starter-data.json new file mode 100644 index 0000000..a3b89ad --- /dev/null +++ b/pilot/agriculture/registry/examples/inputs/starter-data.json @@ -0,0 +1,66 @@ +{ + "river": { + "localIdentifier": "SYNTHETIC-RIVER", + "name": "River Holding" + }, + "landless": { + "localIdentifier": "SYNTHETIC-LANDLESS", + "name": "Landless Cooperative Holding" + }, + "collective": { + "localIdentifier": "SYNTHETIC-COLLECTIVE", + "name": "Hill Collective Holding" + }, + "ana": { + "localIdentifier": "SYNTHETIC-ANA", + "name": "Ana Example" + }, + "bo": { + "localIdentifier": "SYNTHETIC-BO", + "name": "Bo Example" + }, + "cooperative": { + "localIdentifier": "SYNTHETIC-COOPERATIVE", + "name": "Example Agricultural Cooperative" + }, + "group": { + "localIdentifier": "SYNTHETIC-GROUP", + "name": "Hill Growers Group" + }, + "ana-role": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorPerson": { + "recordRef": "ana" + }, + "startDate": "2026-01-01" + }, + "bo-role": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorPerson": { + "recordRef": "bo" + }, + "startDate": "2026-01-01" + }, + "cooperative-role": { + "operatedHolding": { + "recordRef": "landless" + }, + "holdingOperatorOrganization": { + "recordRef": "cooperative" + }, + "startDate": "2026-01-01" + }, + "group-role": { + "operatedHolding": { + "recordRef": "collective" + }, + "holdingOperatorGroup": { + "recordRef": "group" + }, + "startDate": "2026-01-01" + } +} diff --git a/pilot/agriculture/registry/examples/scenarios.json b/pilot/agriculture/registry/examples/scenarios.json new file mode 100644 index 0000000..e1d772d --- /dev/null +++ b/pilot/agriculture/registry/examples/scenarios.json @@ -0,0 +1,199 @@ +{ + "version": 1, + "scenarios": [ + { + "id": "starter-data", + "description": "Create synthetic records and typed relationships through ordinary authorized writes.", + "input": "examples/inputs/starter-data.json", + "steps": [ + { + "id": "river", + "operation": "create", + "entity": "farm", + "client": "editor", + "accessProfile": "editor", + "input": "river", + "capture": "river" + }, + { + "id": "landless", + "operation": "create", + "entity": "farm", + "client": "editor", + "accessProfile": "editor", + "input": "landless", + "capture": "landless" + }, + { + "id": "collective", + "operation": "create", + "entity": "farm", + "client": "editor", + "accessProfile": "editor", + "input": "collective", + "capture": "collective" + }, + { + "id": "ana", + "operation": "create", + "entity": "person", + "client": "editor", + "accessProfile": "editor", + "input": "ana", + "capture": "ana" + }, + { + "id": "bo", + "operation": "create", + "entity": "person", + "client": "editor", + "accessProfile": "editor", + "input": "bo", + "capture": "bo" + }, + { + "id": "cooperative", + "operation": "create", + "entity": "organization", + "client": "editor", + "accessProfile": "editor", + "input": "cooperative", + "capture": "cooperative" + }, + { + "id": "group", + "operation": "create", + "entity": "informal-group", + "client": "editor", + "accessProfile": "editor", + "input": "group", + "capture": "group" + }, + { + "id": "ana-role", + "operation": "create", + "entity": "person-holding-operator-role", + "client": "editor", + "accessProfile": "editor", + "input": "ana-role", + "capture": "ana-role" + }, + { + "id": "bo-role", + "operation": "create", + "entity": "person-holding-operator-role", + "client": "editor", + "accessProfile": "editor", + "input": "bo-role", + "capture": "bo-role" + }, + { + "id": "cooperative-role", + "operation": "create", + "entity": "organization-holding-operator-role", + "client": "editor", + "accessProfile": "editor", + "input": "cooperative-role", + "capture": "cooperative-role" + }, + { + "id": "group-role", + "operation": "create", + "entity": "group-holding-operator-role", + "client": "editor", + "accessProfile": "editor", + "input": "group-role", + "capture": "group-role" + } + ] + }, + { + "id": "first-record", + "description": "Create an independent record without loading samples. Edit its input before starting.", + "input": "examples/inputs/first-record.json", + "steps": [ + { + "id": "create", + "operation": "create", + "entity": "farm", + "client": "editor", + "accessProfile": "editor", + "input": "record", + "capture": "first-record" + }, + { + "id": "get", + "operation": "get", + "entity": "farm", + "client": "editor", + "accessProfile": "editor", + "record": "first-record" + } + ] + }, + { + "id": "reviewed-change", + "description": "Submit, inspect, approve or reject, apply, and inspect history. Approval alone changes no record.", + "input": "examples/inputs/reviewed-change.json", + "steps": [ + { + "id": "draft", + "operation": "create", + "entity": "name-correction", + "client": "editor", + "accessProfile": "editor", + "input": "change", + "capture": "request" + }, + { + "id": "submit", + "operation": "submit", + "entity": "name-correction", + "client": "editor", + "accessProfile": "editor", + "record": "request" + }, + { + "id": "inspect", + "operation": "get", + "entity": "name-correction", + "client": "reviewer", + "accessProfile": "reviewer", + "record": "request" + }, + { + "id": "approve", + "operation": "approve", + "entity": "name-correction", + "client": "reviewer", + "accessProfile": "reviewer", + "record": "request" + }, + { + "id": "reject", + "operation": "reject", + "entity": "name-correction", + "client": "reviewer", + "accessProfile": "reviewer", + "record": "request" + }, + { + "id": "apply", + "operation": "apply", + "entity": "name-correction", + "client": "reviewer", + "accessProfile": "reviewer", + "record": "request" + }, + { + "id": "history", + "operation": "history", + "entity": "farm", + "client": "reviewer", + "accessProfile": "reviewer", + "record": "first-record" + } + ] + } + ] +} diff --git a/pilot/agriculture/registry/registry.yaml b/pilot/agriculture/registry/registry.yaml new file mode 100644 index 0000000..d7bcfa3 --- /dev/null +++ b/pilot/agriculture/registry/registry.yaml @@ -0,0 +1,1067 @@ +{ + "apiVersion": "registry.registrystack.org/v1alpha1", + "kind": "RegistryProject", + "registry": { + "id": "agricultural-holdings", + "canonicalBaseIri": "https://agricultural-holdings.example.org", + "version": "0.1.0", + "defaultLanguage": "en" + }, + "package": { + "environment": "local", + "instanceId": "agriculture-openfn-pilot", + "sequence": 1, + "sourceRevision": "agriculture-openfn-pilot-v1" + }, + "manifestProjection": { + "accessProfile": "editor", + "classificationCeiling": "restricted", + "catalog": { + "baseUrl": "https://agricultural-holdings.example.org", + "title": "Agricultural holding register", + "description": "Maintain agricultural production units and separately asserted holder responsibilities.", + "publisher": { + "id": "synthetic-custodian", + "name": "Synthetic example custodian" + } + }, + "publicService": { + "id": "directory-service", + "title": "Agricultural holding register" + }, + "datasets": [ + { + "id": "directory", + "title": "Agricultural holding register", + "description": "Maintain agricultural production units and separately asserted holder responsibilities.", + "owner": "Synthetic example custodian", + "status": "active" + } + ], + "dataServices": [ + { + "id": "directory-api", + "title": "Agricultural holding register", + "endpointUrl": "https://agricultural-holdings.example.org", + "servesDatasets": [ + "directory" + ] + } + ], + "distributions": [], + "entities": [ + { + "id": "farm", + "title": "Farm", + "conceptUri": "https://publicschema.org/agri/Farm", + "identifiers": [ + { + "field": "local-identifier", + "kind": "local" + } + ], + "fields": [ + { + "id": "name", + "concepts": [ + "https://publicschema.org/name" + ] + } + ] + }, + { + "id": "person", + "title": "Person", + "conceptUri": "https://publicschema.org/Person", + "identifiers": [ + { + "field": "local-identifier", + "kind": "local" + } + ], + "fields": [ + { + "id": "name", + "concepts": [ + "https://publicschema.org/name" + ] + } + ] + }, + { + "id": "organization", + "title": "Organization", + "conceptUri": "https://publicschema.org/Organization", + "identifiers": [ + { + "field": "local-identifier", + "kind": "local" + } + ], + "fields": [ + { + "id": "name", + "concepts": [ + "https://publicschema.org/name" + ] + } + ] + }, + { + "id": "informal-group", + "title": "Informal Group", + "conceptUri": "https://publicschema.org/InformalGroup", + "identifiers": [ + { + "field": "local-identifier", + "kind": "local" + } + ], + "fields": [ + { + "id": "name", + "concepts": [ + "https://publicschema.org/name" + ] + } + ] + }, + { + "id": "person-holding-operator-role", + "title": "Person Holding Operator Role", + "conceptUri": "https://publicschema.org/agri/PersonHoldingOperatorRole", + "identifiers": [], + "fields": [ + { + "id": "operated-holding", + "relationshipRole": "operated-holding", + "relationshipConceptUri": "https://publicschema.org/agri/operated_holding" + }, + { + "id": "holding-operator-person", + "relationshipRole": "holding-operator-person", + "relationshipConceptUri": "https://publicschema.org/agri/holding_operator_person" + }, + { + "id": "start-date", + "concepts": [ + "https://publicschema.org/start_date" + ] + }, + { + "id": "end-date", + "concepts": [ + "https://publicschema.org/end_date" + ] + } + ] + }, + { + "id": "organization-holding-operator-role", + "title": "Organization Holding Operator Role", + "conceptUri": "https://publicschema.org/agri/OrganizationHoldingOperatorRole", + "identifiers": [], + "fields": [ + { + "id": "operated-holding", + "relationshipRole": "operated-holding", + "relationshipConceptUri": "https://publicschema.org/agri/operated_holding" + }, + { + "id": "holding-operator-organization", + "relationshipRole": "holding-operator-organization", + "relationshipConceptUri": "https://publicschema.org/agri/holding_operator_organization" + }, + { + "id": "start-date", + "concepts": [ + "https://publicschema.org/start_date" + ] + }, + { + "id": "end-date", + "concepts": [ + "https://publicschema.org/end_date" + ] + } + ] + }, + { + "id": "group-holding-operator-role", + "title": "Group Holding Operator Role", + "conceptUri": "https://publicschema.org/agri/GroupHoldingOperatorRole", + "identifiers": [], + "fields": [ + { + "id": "operated-holding", + "relationshipRole": "operated-holding", + "relationshipConceptUri": "https://publicschema.org/agri/operated_holding" + }, + { + "id": "holding-operator-group", + "relationshipRole": "holding-operator-group", + "relationshipConceptUri": "https://publicschema.org/agri/holding_operator_group" + }, + { + "id": "start-date", + "concepts": [ + "https://publicschema.org/start_date" + ] + }, + { + "id": "end-date", + "concepts": [ + "https://publicschema.org/end_date" + ] + } + ] + } + ] + }, + "entities": [ + { + "id": "farm", + "primaryDataset": "directory", + "route": "farms", + "mutationMode": "mutable", + "classification": "restricted", + "fields": [ + { + "id": "local-identifier", + "type": "string", + "required": true, + "classification": "restricted", + "apiName": "localIdentifier", + "maxLength": 160, + "minLength": 1 + }, + { + "id": "name", + "type": "string", + "required": true, + "classification": "restricted", + "maxLength": 160, + "minLength": 1 + } + ], + "changeControl": { + "requiredFor": [ + "patch" + ] + }, + "constraints": [ + { + "kind": "unique", + "fields": [ + "local-identifier" + ] + } + ], + "selectorProfiles": [ + { + "id": "by-local-identifier", + "fields": [ + "local-identifier" + ] + } + ], + "events": [ + { + "id": "farm-created-v1", + "trigger": "created", + "projection": [ + "local-identifier" + ], + "webhook": { + "destinationId": "openfn" + } + }, + { + "id": "farm-patched-v1", + "trigger": "patched", + "projection": [ + "local-identifier" + ], + "webhook": { + "destinationId": "openfn" + } + } + ] + }, + { + "id": "person", + "primaryDataset": "directory", + "route": "persons", + "mutationMode": "mutable", + "classification": "restricted", + "fields": [ + { + "id": "local-identifier", + "type": "string", + "required": true, + "classification": "restricted", + "apiName": "localIdentifier", + "maxLength": 160, + "minLength": 1 + }, + { + "id": "name", + "type": "string", + "required": true, + "classification": "restricted", + "maxLength": 160, + "minLength": 1 + } + ], + "constraints": [ + { + "kind": "unique", + "fields": [ + "local-identifier" + ] + } + ] + }, + { + "id": "organization", + "primaryDataset": "directory", + "route": "organizations", + "mutationMode": "mutable", + "classification": "restricted", + "fields": [ + { + "id": "local-identifier", + "type": "string", + "required": true, + "classification": "restricted", + "apiName": "localIdentifier", + "maxLength": 160, + "minLength": 1 + }, + { + "id": "name", + "type": "string", + "required": true, + "classification": "restricted", + "maxLength": 160, + "minLength": 1 + } + ], + "constraints": [ + { + "kind": "unique", + "fields": [ + "local-identifier" + ] + } + ] + }, + { + "id": "informal-group", + "primaryDataset": "directory", + "route": "informal-groups", + "mutationMode": "mutable", + "classification": "restricted", + "fields": [ + { + "id": "local-identifier", + "type": "string", + "required": true, + "classification": "restricted", + "apiName": "localIdentifier", + "maxLength": 160, + "minLength": 1 + }, + { + "id": "name", + "type": "string", + "required": true, + "classification": "restricted", + "maxLength": 160, + "minLength": 1 + } + ], + "constraints": [ + { + "kind": "unique", + "fields": [ + "local-identifier" + ] + } + ] + }, + { + "id": "person-holding-operator-role", + "primaryDataset": "directory", + "route": "person-holding-operator-roles", + "mutationMode": "mutable", + "classification": "restricted", + "fields": [ + { + "id": "operated-holding", + "type": "reference", + "required": true, + "classification": "restricted", + "target": "farm", + "apiName": "operatedHolding" + }, + { + "id": "holding-operator-person", + "type": "reference", + "required": true, + "classification": "restricted", + "target": "person", + "apiName": "holdingOperatorPerson" + }, + { + "id": "start-date", + "type": "date", + "required": false, + "classification": "restricted", + "apiName": "startDate" + }, + { + "id": "end-date", + "type": "date", + "required": false, + "classification": "restricted", + "apiName": "endDate" + } + ], + "constraints": [ + { + "kind": "compare", + "left": "start-date", + "operator": "less_than_or_equal", + "right": "end-date" + } + ] + }, + { + "id": "organization-holding-operator-role", + "primaryDataset": "directory", + "route": "organization-holding-operator-roles", + "mutationMode": "mutable", + "classification": "restricted", + "fields": [ + { + "id": "operated-holding", + "type": "reference", + "required": true, + "classification": "restricted", + "target": "farm", + "apiName": "operatedHolding" + }, + { + "id": "holding-operator-organization", + "type": "reference", + "required": true, + "classification": "restricted", + "target": "organization", + "apiName": "holdingOperatorOrganization" + }, + { + "id": "start-date", + "type": "date", + "required": false, + "classification": "restricted", + "apiName": "startDate" + }, + { + "id": "end-date", + "type": "date", + "required": false, + "classification": "restricted", + "apiName": "endDate" + } + ], + "constraints": [ + { + "kind": "compare", + "left": "start-date", + "operator": "less_than_or_equal", + "right": "end-date" + } + ] + }, + { + "id": "group-holding-operator-role", + "primaryDataset": "directory", + "route": "group-holding-operator-roles", + "mutationMode": "mutable", + "classification": "restricted", + "fields": [ + { + "id": "operated-holding", + "type": "reference", + "required": true, + "classification": "restricted", + "target": "farm", + "apiName": "operatedHolding" + }, + { + "id": "holding-operator-group", + "type": "reference", + "required": true, + "classification": "restricted", + "target": "informal-group", + "apiName": "holdingOperatorGroup" + }, + { + "id": "start-date", + "type": "date", + "required": false, + "classification": "restricted", + "apiName": "startDate" + }, + { + "id": "end-date", + "type": "date", + "required": false, + "classification": "restricted", + "apiName": "endDate" + } + ], + "constraints": [ + { + "kind": "compare", + "left": "start-date", + "operator": "less_than_or_equal", + "right": "end-date" + } + ] + }, + { + "id": "name-correction", + "primaryDataset": "directory", + "route": "name-corrections", + "mutationMode": "mutable", + "classification": "restricted", + "fields": [ + { + "id": "record", + "type": "reference", + "required": true, + "classification": "restricted", + "target": "farm" + }, + { + "id": "name", + "type": "string", + "required": true, + "classification": "restricted", + "maxLength": 160, + "minLength": 1 + }, + { + "id": "reason", + "type": "string", + "required": true, + "classification": "restricted", + "maxLength": 500, + "minLength": 1 + }, + { + "id": "supporting-reference", + "type": "string", + "required": true, + "classification": "restricted", + "apiName": "supportingReference", + "maxLength": 500, + "minLength": 1 + } + ], + "changeRequest": { + "effects": [ + { + "target": { + "fromField": "record" + }, + "operation": "patch", + "set": { + "name": { + "fromField": "name" + } + } + } + ], + "review": { + "stages": [ + { + "id": "review", + "approvals": 1, + "excludeSubmitter": true + } + ] + }, + "application": { + "mode": "manual" + }, + "retention": { + "mode": "operator_erase" + } + } + } + ], + "accessProfiles": [ + { + "id": "reader", + "default": false, + "principalClaim": "registry_principal", + "requiredScopes": [ + "starter:reader" + ], + "requiredPurposes": [ + "starter-learning" + ], + "grants": [ + { + "entity": "farm", + "operations": [ + "get", + "list" + ], + "readableFields": [ + "local-identifier", + "name" + ], + "rowBoundaries": [] + } + ] + }, + { + "id": "editor", + "default": true, + "principalClaim": "registry_principal", + "requiredScopes": [ + "starter:editor" + ], + "requiredPurposes": [ + "starter-learning" + ], + "grants": [ + { + "entity": "farm", + "operations": [ + "get", + "list", + "revisions", + "create" + ], + "readableFields": [ + "local-identifier", + "name" + ], + "rowBoundaries": [], + "revisionAccess": true, + "writableFields": [ + "local-identifier", + "name" + ] + }, + { + "entity": "person", + "operations": [ + "get", + "list", + "revisions", + "create", + "patch" + ], + "readableFields": [ + "local-identifier", + "name" + ], + "rowBoundaries": [], + "revisionAccess": true, + "writableFields": [ + "local-identifier", + "name" + ] + }, + { + "entity": "organization", + "operations": [ + "get", + "list", + "revisions", + "create", + "patch" + ], + "readableFields": [ + "local-identifier", + "name" + ], + "rowBoundaries": [], + "revisionAccess": true, + "writableFields": [ + "local-identifier", + "name" + ] + }, + { + "entity": "informal-group", + "operations": [ + "get", + "list", + "revisions", + "create", + "patch" + ], + "readableFields": [ + "local-identifier", + "name" + ], + "rowBoundaries": [], + "revisionAccess": true, + "writableFields": [ + "local-identifier", + "name" + ] + }, + { + "entity": "person-holding-operator-role", + "operations": [ + "get", + "list", + "revisions", + "create", + "patch" + ], + "readableFields": [ + "operated-holding", + "holding-operator-person", + "start-date", + "end-date" + ], + "rowBoundaries": [], + "revisionAccess": true, + "writableFields": [ + "operated-holding", + "holding-operator-person", + "start-date", + "end-date" + ] + }, + { + "entity": "organization-holding-operator-role", + "operations": [ + "get", + "list", + "revisions", + "create", + "patch" + ], + "readableFields": [ + "operated-holding", + "holding-operator-organization", + "start-date", + "end-date" + ], + "rowBoundaries": [], + "revisionAccess": true, + "writableFields": [ + "operated-holding", + "holding-operator-organization", + "start-date", + "end-date" + ] + }, + { + "entity": "group-holding-operator-role", + "operations": [ + "get", + "list", + "revisions", + "create", + "patch" + ], + "readableFields": [ + "operated-holding", + "holding-operator-group", + "start-date", + "end-date" + ], + "rowBoundaries": [], + "revisionAccess": true, + "writableFields": [ + "operated-holding", + "holding-operator-group", + "start-date", + "end-date" + ] + }, + { + "entity": "name-correction", + "operations": [ + "create", + "get", + "patch", + "submit_request", + "revise_request", + "cancel_request" + ], + "readableFields": [ + "record", + "name", + "reason", + "supporting-reference" + ], + "writableFields": [ + "record", + "name", + "reason", + "supporting-reference" + ], + "requestVisibility": "owner", + "rowBoundaries": [] + } + ] + }, + { + "id": "reviewer", + "default": false, + "principalClaim": "registry_principal", + "requiredScopes": [ + "starter:reviewer" + ], + "requiredPurposes": [ + "starter-learning" + ], + "grants": [ + { + "entity": "farm", + "operations": [ + "get", + "list", + "revisions" + ], + "readableFields": [ + "local-identifier", + "name" + ], + "rowBoundaries": [], + "revisionAccess": true + }, + { + "entity": "person", + "operations": [ + "get", + "list", + "revisions" + ], + "readableFields": [ + "local-identifier", + "name" + ], + "rowBoundaries": [], + "revisionAccess": true + }, + { + "entity": "organization", + "operations": [ + "get", + "list", + "revisions" + ], + "readableFields": [ + "local-identifier", + "name" + ], + "rowBoundaries": [], + "revisionAccess": true + }, + { + "entity": "informal-group", + "operations": [ + "get", + "list", + "revisions" + ], + "readableFields": [ + "local-identifier", + "name" + ], + "rowBoundaries": [], + "revisionAccess": true + }, + { + "entity": "person-holding-operator-role", + "operations": [ + "get", + "list", + "revisions" + ], + "readableFields": [ + "operated-holding", + "holding-operator-person", + "start-date", + "end-date" + ], + "rowBoundaries": [], + "revisionAccess": true + }, + { + "entity": "organization-holding-operator-role", + "operations": [ + "get", + "list", + "revisions" + ], + "readableFields": [ + "operated-holding", + "holding-operator-organization", + "start-date", + "end-date" + ], + "rowBoundaries": [], + "revisionAccess": true + }, + { + "entity": "group-holding-operator-role", + "operations": [ + "get", + "list", + "revisions" + ], + "readableFields": [ + "operated-holding", + "holding-operator-group", + "start-date", + "end-date" + ], + "rowBoundaries": [], + "revisionAccess": true + }, + { + "entity": "name-correction", + "operations": [ + "get", + "approve_request", + "reject_request", + "request_revision", + "apply_request" + ], + "readableFields": [ + "record", + "name", + "reason", + "supporting-reference" + ], + "rowBoundaries": [], + "reviewStages": [ + { + "stage": "review", + "targets": [ + { + "entity": "farm", + "readableFields": [ + "local-identifier", + "name" + ], + "rowBoundaries": [] + } + ] + } + ], + "applyTargets": [ + { + "entity": "farm", + "rowBoundaries": [] + } + ] + } + ] + }, + { + "id": "openfn-service", + "default": false, + "principalClaim": "registry_principal", + "requiredScopes": [ + "pilot:submit" + ], + "requiredPurposes": [ + "agriculture-registration" + ], + "grants": [ + { + "entity": "farm", + "operations": [ + "get", + "create", + "lookup" + ], + "readableFields": [ + "local-identifier", + "name" + ], + "rowBoundaries": [], + "writableFields": [ + "local-identifier", + "name" + ], + "lookups": [ + { + "selector": "by-local-identifier", + "valueOrigin": "request" + } + ] + }, + { + "entity": "name-correction", + "operations": [ + "create", + "get", + "patch", + "submit_request", + "revise_request", + "cancel_request" + ], + "readableFields": [ + "record", + "name", + "reason", + "supporting-reference" + ], + "writableFields": [ + "record", + "name", + "reason", + "supporting-reference" + ], + "requestVisibility": "owner", + "rowBoundaries": [] + } + ] + }, + { + "id": "evidence-source", + "principalClaim": "registry_principal", + "requiredScopes": [ + "pilot:evidence-source" + ], + "requiredPurposes": [ + "evidence-source-read" + ], + "grants": [ + { + "entity": "farm", + "operations": [ + "lookup" + ], + "readableFields": [ + "local-identifier" + ], + "lookups": [ + { + "selector": "by-local-identifier", + "valueOrigin": "request" + } + ], + "rowBoundaries": [] + } + ], + "default": true + } + ] +} diff --git a/pilot/agriculture/registry/starter-template.json b/pilot/agriculture/registry/starter-template.json new file mode 100644 index 0000000..05d4756 --- /dev/null +++ b/pilot/agriculture/registry/starter-template.json @@ -0,0 +1,28 @@ +{ + "version": 1, + "starterId": "agricultural-holdings", + "variantId": "core", + "templateVersion": "0.1.0", + "title": "Agricultural holding register", + "description": "Maintain agricultural production units and separately asserted holder responsibilities.", + "firstRecordEntity": "farm", + "publicSchema": { + "repository": "https://github.com/PublicSchema/publicschema.org", + "commit": "1ea9ce333918693b29aec31068fac412e02cb8dc", + "status": "draft", + "license": "CC-BY-4.0" + }, + "review": { + "entity": "name-correction", + "field": "name", + "submitter": "editor", + "reviewer": "reviewer", + "applier": "reviewer", + "description": "A distinct reviewer approves a fixed name correction. Approval leaves the record unchanged until the reviewer applies the frozen effect." + }, + "scopeNotes": [ + "A holding is a production unit; registration establishes neither land title nor benefit entitlement.", + "Readers see holdings only. Identity records and typed person, organization or informal-group holder responsibilities are restricted to editors and reviewers.", + "Holder responsibilities are separate assertions, including joint holders; they do not imply daily management or work." + ] +} diff --git a/pilot/agriculture/registry/tests/journeys.yaml b/pilot/agriculture/registry/tests/journeys.yaml new file mode 100644 index 0000000..3cbaab5 --- /dev/null +++ b/pilot/agriculture/registry/tests/journeys.yaml @@ -0,0 +1,1192 @@ +{ + "apiVersion": "registry.registrystack.org/breg-journeys/v1", + "journeys": [ + { + "id": "starter-model-policy", + "steps": [ + { + "id": "seed-river", + "entity": "farm", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "localIdentifier": "SYNTHETIC-RIVER", + "name": "River Holding" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "river" + }, + { + "id": "seed-landless", + "entity": "farm", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "localIdentifier": "SYNTHETIC-LANDLESS", + "name": "Landless Cooperative Holding" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "landless" + }, + { + "id": "seed-collective", + "entity": "farm", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "localIdentifier": "SYNTHETIC-COLLECTIVE", + "name": "Hill Collective Holding" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "collective" + }, + { + "id": "seed-ana", + "entity": "person", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "localIdentifier": "SYNTHETIC-ANA", + "name": "Ana Example" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "ana" + }, + { + "id": "seed-bo", + "entity": "person", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "localIdentifier": "SYNTHETIC-BO", + "name": "Bo Example" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "bo" + }, + { + "id": "seed-cooperative", + "entity": "organization", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "localIdentifier": "SYNTHETIC-COOPERATIVE", + "name": "Example Agricultural Cooperative" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "cooperative" + }, + { + "id": "seed-group", + "entity": "informal-group", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "localIdentifier": "SYNTHETIC-GROUP", + "name": "Hill Growers Group" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "group" + }, + { + "id": "seed-ana-role", + "entity": "person-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorPerson": { + "recordRef": "ana" + }, + "startDate": "2026-01-01" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "ana-role" + }, + { + "id": "seed-bo-role", + "entity": "person-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorPerson": { + "recordRef": "bo" + }, + "startDate": "2026-01-01" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "bo-role" + }, + { + "id": "seed-cooperative-role", + "entity": "organization-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "landless" + }, + "holdingOperatorOrganization": { + "recordRef": "cooperative" + }, + "startDate": "2026-01-01" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "cooperative-role" + }, + { + "id": "seed-group-role", + "entity": "group-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "collective" + }, + "holdingOperatorGroup": { + "recordRef": "group" + }, + "startDate": "2026-01-01" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "group-role" + }, + { + "id": "first-record", + "entity": "farm", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "localIdentifier": "JOURNEY-FIRST-001", + "name": "Frist Record" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "first-record" + }, + { + "id": "draft-correction", + "entity": "name-correction", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "record": { + "recordRef": "first-record" + }, + "name": "First Record", + "reason": "Synthetic spelling correction", + "supportingReference": "SYNTHETIC-SUPPORT-001" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "draft" + }, + { + "id": "before-submit", + "entity": "name-correction", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "draft" + }, + "expect": { + "outcome": "success", + "status": 200 + }, + "capture": "before-submit" + }, + { + "id": "submit-correction", + "entity": "name-correction", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "submit_request", + "etagRef": "before-submit", + "recordRef": "before-submit" + }, + "expect": { + "outcome": "success", + "status": 200 + } + }, + { + "id": "review", + "entity": "name-correction", + "accessProfile": "reviewer", + "claims": { + "principal": "synthetic-reviewer", + "scopes": [ + "starter:reviewer" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "draft" + }, + "expect": { + "outcome": "success", + "status": 200 + }, + "capture": "review" + }, + { + "id": "approve-correction", + "entity": "name-correction", + "accessProfile": "reviewer", + "claims": { + "principal": "synthetic-reviewer", + "scopes": [ + "starter:reviewer" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "approve_request", + "etagRef": "review", + "proposalVersionRef": "review", + "effectDigestRef": "review", + "stage": "review", + "recordRef": "review" + }, + "expect": { + "outcome": "success", + "status": 200 + } + }, + { + "id": "approval-does-not-apply", + "entity": "farm", + "accessProfile": "reader", + "claims": { + "principal": "synthetic-reader", + "scopes": [ + "starter:reader" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "first-record" + }, + "expect": { + "outcome": "success", + "status": 200, + "fields": { + "name": "Frist Record", + "localIdentifier": "JOURNEY-FIRST-001" + } + }, + "capture": "approval-does-not-apply" + }, + { + "id": "before-apply", + "entity": "name-correction", + "accessProfile": "reviewer", + "claims": { + "principal": "synthetic-reviewer", + "scopes": [ + "starter:reviewer" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "draft" + }, + "expect": { + "outcome": "success", + "status": 200 + }, + "capture": "before-apply" + }, + { + "id": "apply-correction", + "entity": "name-correction", + "accessProfile": "reviewer", + "claims": { + "principal": "synthetic-reviewer", + "scopes": [ + "starter:reviewer" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "apply_request", + "etagRef": "before-apply", + "proposalVersionRef": "before-apply", + "effectDigestRef": "before-apply", + "recordRef": "before-apply" + }, + "expect": { + "outcome": "success", + "status": 200 + } + }, + { + "id": "corrected-record", + "entity": "farm", + "accessProfile": "reader", + "claims": { + "principal": "synthetic-reader", + "scopes": [ + "starter:reader" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "first-record" + }, + "expect": { + "outcome": "success", + "status": 200, + "fields": { + "name": "First Record", + "localIdentifier": "JOURNEY-FIRST-001" + } + }, + "capture": "corrected-record" + }, + { + "id": "unchanged-ana-role", + "entity": "person-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "ana-role" + }, + "expect": { + "outcome": "success", + "status": 200, + "fields": { + "startDate": "2026-01-01" + } + }, + "capture": "unchanged-ana-role" + }, + { + "id": "unchanged-bo-role", + "entity": "person-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "bo-role" + }, + "expect": { + "outcome": "success", + "status": 200, + "fields": { + "startDate": "2026-01-01" + } + }, + "capture": "unchanged-bo-role" + }, + { + "id": "unchanged-cooperative-role", + "entity": "organization-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "cooperative-role" + }, + "expect": { + "outcome": "success", + "status": 200, + "fields": { + "startDate": "2026-01-01" + } + }, + "capture": "unchanged-cooperative-role" + }, + { + "id": "unchanged-group-role", + "entity": "group-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "group-role" + }, + "expect": { + "outcome": "success", + "status": 200, + "fields": { + "startDate": "2026-01-01" + } + }, + "capture": "unchanged-group-role" + }, + { + "id": "rejected-draft", + "entity": "name-correction", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "record": { + "recordRef": "first-record" + }, + "name": "Rejected Record", + "reason": "Synthetic rejection exercise", + "supportingReference": "SYNTHETIC-SUPPORT-002" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "reject-draft" + }, + { + "id": "reject-before-submit", + "entity": "name-correction", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "reject-draft" + }, + "expect": { + "outcome": "success", + "status": 200 + }, + "capture": "reject-before-submit" + }, + { + "id": "submit-rejected", + "entity": "name-correction", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "submit_request", + "etagRef": "reject-before-submit", + "recordRef": "reject-before-submit" + }, + "expect": { + "outcome": "success", + "status": 200 + } + }, + { + "id": "reject-before-decision", + "entity": "name-correction", + "accessProfile": "reviewer", + "claims": { + "principal": "synthetic-reviewer", + "scopes": [ + "starter:reviewer" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "reject-draft" + }, + "expect": { + "outcome": "success", + "status": 200 + }, + "capture": "reject-before-decision" + }, + { + "id": "reject-correction", + "entity": "name-correction", + "accessProfile": "reviewer", + "claims": { + "principal": "synthetic-reviewer", + "scopes": [ + "starter:reviewer" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "reject_request", + "etagRef": "reject-before-decision", + "proposalVersionRef": "reject-before-decision", + "effectDigestRef": "reject-before-decision", + "stage": "review", + "recordRef": "reject-before-decision" + }, + "expect": { + "outcome": "success", + "status": 200 + } + }, + { + "id": "rejection-preserves-record", + "entity": "farm", + "accessProfile": "reader", + "claims": { + "principal": "synthetic-reader", + "scopes": [ + "starter:reader" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "first-record" + }, + "expect": { + "outcome": "success", + "status": 200, + "fields": { + "name": "First Record", + "localIdentifier": "JOURNEY-FIRST-001" + } + }, + "capture": "rejection-preserves-record" + }, + { + "id": "person-unknown-dates", + "entity": "person-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorPerson": { + "recordRef": "ana" + } + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "person-unknown" + }, + { + "id": "person-equal-dates", + "entity": "person-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorPerson": { + "recordRef": "ana" + }, + "startDate": "2026-01-01", + "endDate": "2026-01-01" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "person-equal" + }, + { + "id": "person-reversed-dates-refused", + "entity": "person-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorPerson": { + "recordRef": "ana" + }, + "startDate": "2026-02-01", + "endDate": "2026-01-01" + } + }, + "expect": { + "outcome": "refusal", + "status": 409, + "problemCode": "mutation.conflict" + } + }, + { + "id": "person-malformed-date-refused", + "entity": "person-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorPerson": { + "recordRef": "ana" + }, + "startDate": "2026-02-30" + } + }, + "expect": { + "outcome": "refusal", + "status": 400, + "problemCode": "request.invalid" + } + }, + { + "id": "organization-unknown-dates", + "entity": "organization-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorOrganization": { + "recordRef": "cooperative" + } + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "organization-unknown" + }, + { + "id": "organization-equal-dates", + "entity": "organization-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorOrganization": { + "recordRef": "cooperative" + }, + "startDate": "2026-01-01", + "endDate": "2026-01-01" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "organization-equal" + }, + { + "id": "organization-reversed-dates-refused", + "entity": "organization-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorOrganization": { + "recordRef": "cooperative" + }, + "startDate": "2026-02-01", + "endDate": "2026-01-01" + } + }, + "expect": { + "outcome": "refusal", + "status": 409, + "problemCode": "mutation.conflict" + } + }, + { + "id": "organization-malformed-date-refused", + "entity": "organization-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorOrganization": { + "recordRef": "cooperative" + }, + "startDate": "2026-02-30" + } + }, + "expect": { + "outcome": "refusal", + "status": 400, + "problemCode": "request.invalid" + } + }, + { + "id": "group-unknown-dates", + "entity": "group-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorGroup": { + "recordRef": "group" + } + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "group-unknown" + }, + { + "id": "group-equal-dates", + "entity": "group-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorGroup": { + "recordRef": "group" + }, + "startDate": "2026-01-01", + "endDate": "2026-01-01" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "group-equal" + }, + { + "id": "group-reversed-dates-refused", + "entity": "group-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorGroup": { + "recordRef": "group" + }, + "startDate": "2026-02-01", + "endDate": "2026-01-01" + } + }, + "expect": { + "outcome": "refusal", + "status": 409, + "problemCode": "mutation.conflict" + } + }, + { + "id": "group-malformed-date-refused", + "entity": "group-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorGroup": { + "recordRef": "group" + }, + "startDate": "2026-02-30" + } + }, + "expect": { + "outcome": "refusal", + "status": 400, + "problemCode": "request.invalid" + } + } + ] + }, + { + "id": "integration-permission-ceiling", + "steps": [ + { + "id": "service-create", + "entity": "farm", + "accessProfile": "openfn-service", + "claims": { + "principal": "synthetic-openfn-service", + "scopes": [ + "pilot:submit" + ], + "purpose": "agriculture-registration" + }, + "request": { + "operation": "create", + "data": { + "localIdentifier": "SYNTHETIC-SERVICE-001", + "name": "Synthetic Service Holding" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "service-farm" + }, + { + "id": "service-lookup", + "entity": "farm", + "accessProfile": "openfn-service", + "claims": { + "principal": "synthetic-openfn-service", + "scopes": [ + "pilot:submit" + ], + "purpose": "agriculture-registration" + }, + "request": { + "operation": "lookup", + "selector": "by-local-identifier", + "values": { + "localIdentifier": "SYNTHETIC-SERVICE-001" + } + }, + "expect": { + "outcome": "success", + "status": 200 + } + }, + { + "id": "source-lookup", + "entity": "farm", + "accessProfile": "evidence-source", + "claims": { + "principal": "synthetic-evidence-source", + "scopes": [ + "pilot:evidence-source" + ], + "purpose": "evidence-source-read" + }, + "request": { + "operation": "lookup", + "selector": "by-local-identifier", + "values": { + "localIdentifier": "SYNTHETIC-SERVICE-001" + } + }, + "expect": { + "outcome": "success", + "status": 200, + "fields": { + "localIdentifier": "SYNTHETIC-SERVICE-001" + } + } + } + ] + } + ] +} diff --git a/pilot/agriculture/registry/tests/security-journeys.yaml b/pilot/agriculture/registry/tests/security-journeys.yaml new file mode 100644 index 0000000..37503fd --- /dev/null +++ b/pilot/agriculture/registry/tests/security-journeys.yaml @@ -0,0 +1,1218 @@ +{ + "apiVersion": "registry.registrystack.org/breg-journeys/v1", + "journeys": [ + { + "id": "starter-model-policy", + "steps": [ + { + "id": "seed-river", + "entity": "farm", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "localIdentifier": "SYNTHETIC-RIVER", + "name": "River Holding" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "river" + }, + { + "id": "seed-landless", + "entity": "farm", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "localIdentifier": "SYNTHETIC-LANDLESS", + "name": "Landless Cooperative Holding" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "landless" + }, + { + "id": "seed-collective", + "entity": "farm", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "localIdentifier": "SYNTHETIC-COLLECTIVE", + "name": "Hill Collective Holding" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "collective" + }, + { + "id": "seed-ana", + "entity": "person", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "localIdentifier": "SYNTHETIC-ANA", + "name": "Ana Example" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "ana" + }, + { + "id": "seed-bo", + "entity": "person", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "localIdentifier": "SYNTHETIC-BO", + "name": "Bo Example" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "bo" + }, + { + "id": "seed-cooperative", + "entity": "organization", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "localIdentifier": "SYNTHETIC-COOPERATIVE", + "name": "Example Agricultural Cooperative" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "cooperative" + }, + { + "id": "seed-group", + "entity": "informal-group", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "localIdentifier": "SYNTHETIC-GROUP", + "name": "Hill Growers Group" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "group" + }, + { + "id": "seed-ana-role", + "entity": "person-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorPerson": { + "recordRef": "ana" + }, + "startDate": "2026-01-01" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "ana-role" + }, + { + "id": "seed-bo-role", + "entity": "person-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorPerson": { + "recordRef": "bo" + }, + "startDate": "2026-01-01" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "bo-role" + }, + { + "id": "seed-cooperative-role", + "entity": "organization-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "landless" + }, + "holdingOperatorOrganization": { + "recordRef": "cooperative" + }, + "startDate": "2026-01-01" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "cooperative-role" + }, + { + "id": "seed-group-role", + "entity": "group-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "collective" + }, + "holdingOperatorGroup": { + "recordRef": "group" + }, + "startDate": "2026-01-01" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "group-role" + }, + { + "id": "first-record", + "entity": "farm", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "localIdentifier": "JOURNEY-FIRST-001", + "name": "Frist Record" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "first-record" + }, + { + "id": "unscoped-caller-cannot-create", + "entity": "farm", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-reader", + "scopes": [], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "localIdentifier": "DENIED-001", + "name": "Denied" + } + }, + "expect": { + "outcome": "refusal", + "status": 404, + "problemCode": "resource.not_found" + } + }, + { + "id": "missing-scope-cannot-read", + "entity": "farm", + "accessProfile": "reader", + "claims": { + "principal": "synthetic-reader", + "scopes": [], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "first-record" + }, + "expect": { + "outcome": "refusal", + "status": 404, + "problemCode": "resource.not_found" + } + }, + { + "id": "draft-correction", + "entity": "name-correction", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "record": { + "recordRef": "first-record" + }, + "name": "First Record", + "reason": "Synthetic spelling correction", + "supportingReference": "SYNTHETIC-SUPPORT-001" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "draft" + }, + { + "id": "unscoped-caller-cannot-read-review-material", + "entity": "name-correction", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-reader", + "scopes": [], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "draft" + }, + "expect": { + "outcome": "refusal", + "status": 404, + "problemCode": "resource.not_found" + } + }, + { + "id": "before-submit", + "entity": "name-correction", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "draft" + }, + "expect": { + "outcome": "success", + "status": 200 + }, + "capture": "before-submit" + }, + { + "id": "submit-correction", + "entity": "name-correction", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "submit_request", + "etagRef": "before-submit", + "recordRef": "before-submit" + }, + "expect": { + "outcome": "success", + "status": 200 + } + }, + { + "id": "review", + "entity": "name-correction", + "accessProfile": "reviewer", + "claims": { + "principal": "synthetic-reviewer", + "scopes": [ + "starter:reviewer" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "draft" + }, + "expect": { + "outcome": "success", + "status": 200 + }, + "capture": "review" + }, + { + "id": "foreign-reviewer-precondition-refused", + "entity": "name-correction", + "accessProfile": "reviewer", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:reviewer" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "approve_request", + "etagRef": "review", + "proposalVersionRef": "review", + "effectDigestRef": "review", + "stage": "review", + "recordRef": "review" + }, + "expect": { + "outcome": "refusal", + "status": 412, + "problemCode": "precondition.failed" + } + }, + { + "id": "approve-correction", + "entity": "name-correction", + "accessProfile": "reviewer", + "claims": { + "principal": "synthetic-reviewer", + "scopes": [ + "starter:reviewer" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "approve_request", + "etagRef": "review", + "proposalVersionRef": "review", + "effectDigestRef": "review", + "stage": "review", + "recordRef": "review" + }, + "expect": { + "outcome": "success", + "status": 200 + } + }, + { + "id": "approval-does-not-apply", + "entity": "farm", + "accessProfile": "reader", + "claims": { + "principal": "synthetic-reader", + "scopes": [ + "starter:reader" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "first-record" + }, + "expect": { + "outcome": "success", + "status": 200, + "fields": { + "name": "Frist Record", + "localIdentifier": "JOURNEY-FIRST-001" + } + }, + "capture": "approval-does-not-apply" + }, + { + "id": "before-apply", + "entity": "name-correction", + "accessProfile": "reviewer", + "claims": { + "principal": "synthetic-reviewer", + "scopes": [ + "starter:reviewer" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "draft" + }, + "expect": { + "outcome": "success", + "status": 200 + }, + "capture": "before-apply" + }, + { + "id": "apply-correction", + "entity": "name-correction", + "accessProfile": "reviewer", + "claims": { + "principal": "synthetic-reviewer", + "scopes": [ + "starter:reviewer" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "apply_request", + "etagRef": "before-apply", + "proposalVersionRef": "before-apply", + "effectDigestRef": "before-apply", + "recordRef": "before-apply" + }, + "expect": { + "outcome": "success", + "status": 200 + } + }, + { + "id": "corrected-record", + "entity": "farm", + "accessProfile": "reader", + "claims": { + "principal": "synthetic-reader", + "scopes": [ + "starter:reader" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "first-record" + }, + "expect": { + "outcome": "success", + "status": 200, + "fields": { + "name": "First Record", + "localIdentifier": "JOURNEY-FIRST-001" + } + }, + "capture": "corrected-record" + }, + { + "id": "unchanged-ana-role", + "entity": "person-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "ana-role" + }, + "expect": { + "outcome": "success", + "status": 200, + "fields": { + "startDate": "2026-01-01" + } + }, + "capture": "unchanged-ana-role" + }, + { + "id": "unchanged-bo-role", + "entity": "person-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "bo-role" + }, + "expect": { + "outcome": "success", + "status": 200, + "fields": { + "startDate": "2026-01-01" + } + }, + "capture": "unchanged-bo-role" + }, + { + "id": "unchanged-cooperative-role", + "entity": "organization-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "cooperative-role" + }, + "expect": { + "outcome": "success", + "status": 200, + "fields": { + "startDate": "2026-01-01" + } + }, + "capture": "unchanged-cooperative-role" + }, + { + "id": "unchanged-group-role", + "entity": "group-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "group-role" + }, + "expect": { + "outcome": "success", + "status": 200, + "fields": { + "startDate": "2026-01-01" + } + }, + "capture": "unchanged-group-role" + }, + { + "id": "rejected-draft", + "entity": "name-correction", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "record": { + "recordRef": "first-record" + }, + "name": "Rejected Record", + "reason": "Synthetic rejection exercise", + "supportingReference": "SYNTHETIC-SUPPORT-002" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "reject-draft" + }, + { + "id": "reject-before-submit", + "entity": "name-correction", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "reject-draft" + }, + "expect": { + "outcome": "success", + "status": 200 + }, + "capture": "reject-before-submit" + }, + { + "id": "submit-rejected", + "entity": "name-correction", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "submit_request", + "etagRef": "reject-before-submit", + "recordRef": "reject-before-submit" + }, + "expect": { + "outcome": "success", + "status": 200 + } + }, + { + "id": "reject-before-decision", + "entity": "name-correction", + "accessProfile": "reviewer", + "claims": { + "principal": "synthetic-reviewer", + "scopes": [ + "starter:reviewer" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "reject-draft" + }, + "expect": { + "outcome": "success", + "status": 200 + }, + "capture": "reject-before-decision" + }, + { + "id": "reject-correction", + "entity": "name-correction", + "accessProfile": "reviewer", + "claims": { + "principal": "synthetic-reviewer", + "scopes": [ + "starter:reviewer" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "reject_request", + "etagRef": "reject-before-decision", + "proposalVersionRef": "reject-before-decision", + "effectDigestRef": "reject-before-decision", + "stage": "review", + "recordRef": "reject-before-decision" + }, + "expect": { + "outcome": "success", + "status": 200 + } + }, + { + "id": "rejection-preserves-record", + "entity": "farm", + "accessProfile": "reader", + "claims": { + "principal": "synthetic-reader", + "scopes": [ + "starter:reader" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "first-record" + }, + "expect": { + "outcome": "success", + "status": 200, + "fields": { + "name": "First Record", + "localIdentifier": "JOURNEY-FIRST-001" + } + }, + "capture": "rejection-preserves-record" + }, + { + "id": "person-unknown-dates", + "entity": "person-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorPerson": { + "recordRef": "ana" + } + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "person-unknown" + }, + { + "id": "person-equal-dates", + "entity": "person-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorPerson": { + "recordRef": "ana" + }, + "startDate": "2026-01-01", + "endDate": "2026-01-01" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "person-equal" + }, + { + "id": "person-reversed-dates-refused", + "entity": "person-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorPerson": { + "recordRef": "ana" + }, + "startDate": "2026-02-01", + "endDate": "2026-01-01" + } + }, + "expect": { + "outcome": "refusal", + "status": 409, + "problemCode": "mutation.conflict" + } + }, + { + "id": "person-malformed-date-refused", + "entity": "person-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorPerson": { + "recordRef": "ana" + }, + "startDate": "2026-02-30" + } + }, + "expect": { + "outcome": "refusal", + "status": 400, + "problemCode": "request.invalid" + } + }, + { + "id": "organization-unknown-dates", + "entity": "organization-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorOrganization": { + "recordRef": "cooperative" + } + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "organization-unknown" + }, + { + "id": "organization-equal-dates", + "entity": "organization-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorOrganization": { + "recordRef": "cooperative" + }, + "startDate": "2026-01-01", + "endDate": "2026-01-01" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "organization-equal" + }, + { + "id": "organization-reversed-dates-refused", + "entity": "organization-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorOrganization": { + "recordRef": "cooperative" + }, + "startDate": "2026-02-01", + "endDate": "2026-01-01" + } + }, + "expect": { + "outcome": "refusal", + "status": 409, + "problemCode": "mutation.conflict" + } + }, + { + "id": "organization-malformed-date-refused", + "entity": "organization-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorOrganization": { + "recordRef": "cooperative" + }, + "startDate": "2026-02-30" + } + }, + "expect": { + "outcome": "refusal", + "status": 400, + "problemCode": "request.invalid" + } + }, + { + "id": "group-unknown-dates", + "entity": "group-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorGroup": { + "recordRef": "group" + } + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "group-unknown" + }, + { + "id": "group-equal-dates", + "entity": "group-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorGroup": { + "recordRef": "group" + }, + "startDate": "2026-01-01", + "endDate": "2026-01-01" + } + }, + "expect": { + "outcome": "success", + "status": 201 + }, + "capture": "group-equal" + }, + { + "id": "group-reversed-dates-refused", + "entity": "group-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorGroup": { + "recordRef": "group" + }, + "startDate": "2026-02-01", + "endDate": "2026-01-01" + } + }, + "expect": { + "outcome": "refusal", + "status": 409, + "problemCode": "mutation.conflict" + } + }, + { + "id": "group-malformed-date-refused", + "entity": "group-holding-operator-role", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-editor", + "scopes": [ + "starter:editor" + ], + "purpose": "starter-learning" + }, + "request": { + "operation": "create", + "data": { + "operatedHolding": { + "recordRef": "river" + }, + "holdingOperatorGroup": { + "recordRef": "group" + }, + "startDate": "2026-02-30" + } + }, + "expect": { + "outcome": "refusal", + "status": 400, + "problemCode": "request.invalid" + } + }, + { + "id": "unscoped-caller-cannot-read-person", + "entity": "person", + "accessProfile": "editor", + "claims": { + "principal": "synthetic-reader", + "scopes": [], + "purpose": "starter-learning" + }, + "request": { + "operation": "get", + "recordRef": "ana" + }, + "expect": { + "outcome": "refusal", + "status": 404, + "problemCode": "resource.not_found" + } + } + ] + } + ] +} diff --git a/pilot/agriculture/review.py b/pilot/agriculture/review.py new file mode 100644 index 0000000..951d747 --- /dev/null +++ b/pilot/agriculture/review.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Inspect a synthetic correction, then explicitly approve or apply that snapshot.""" +import argparse +import json +import os +from pathlib import Path +import subprocess +import urllib.request +import urllib.error +import uuid + +ROOT=Path('/config/breg') +ORIGIN='http://127.0.0.1:8090' + + +def token(): + result=subprocess.run(['mint','token','--url','http://127.0.0.1:8091/token','--client-id','reviewer','--key',str(ROOT/'clients/reviewer/signing-p256-private-jwk')],capture_output=True,text=True) + if result.returncode: + raise RuntimeError('Reviewer authentication failed') + return result.stdout.strip() + + +def request(path,bearer,method='GET',body=None,headers=None): + req=urllib.request.Request(ORIGIN+path,method=method,data=None if body is None else json.dumps(body).encode(),headers={'Authorization':'Bearer '+bearer,'Accept':'application/json',**({'Content-Type':'application/json'} if body is not None else {}),**(headers or {})}) + try: + with urllib.request.urlopen(req,timeout=10) as response: + return json.load(response) + except urllib.error.HTTPError as error: + problem=json.load(error) + raise RuntimeError(f"Registry refused ({error.code}): {problem.get('code','unknown')}") from None + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('action',choices=['inspect','approve','apply']) + parser.add_argument('request_id',type=uuid.UUID) + args=parser.parse_args() + os.umask(0o077) + request_id=str(args.request_id) + snapshot=ROOT/'review'/f'{request_id}.json' + bearer=token() + if args.action=='inspect': + value=request(f'/v1/records/name-corrections/{request_id}?accessProfile=reviewer',bearer) + snapshot.parent.mkdir(exist_ok=True,mode=0o700) + snapshot.write_text(json.dumps(value,indent=2)+'\n'); snapshot.chmod(0o600) + print(json.dumps(value,indent=2)) + print('Review the proposed values above. Approval and application are separate commands.') + return + if not snapshot.exists(): + raise RuntimeError('Inspect this correction first; no reviewed snapshot exists') + value=json.loads(snapshot.read_text()) + actions=value['data']['request']['actions'] + operation={'approve':'approve_request','apply':'apply_request'}[args.action] + selected=[action for action in actions if action['operation']==operation] + if len(selected)!=1: + raise RuntimeError('The inspected snapshot has no unique permitted action. Inspect current state first.') + action=selected[0] + suffix='/actions/stages/review/approve' if args.action=='approve' else '/actions/apply' + expected=f'/v1/records/name-corrections/{request_id}'+suffix + if action['href'].split('?')[0]!=expected or action['method']!='POST': + raise RuntimeError('Unexpected action location; refusing to forward a reviewer credential') + body={key:action[key] for key in ['proposalVersion','effectDigest']} + key=f"reviewer:{request_id}:{operation}:{body['proposalVersion']}:{body['effectDigest']}" + result=request(expected+'?accessProfile=reviewer',bearer,'POST',body,{'If-Match':action['ifMatch'],'Idempotency-Key':key}) + print(json.dumps(result,indent=2)) + print('Approval recorded; the farm is unchanged. Inspect again before applying.' if args.action=='approve' else 'The reviewed correction was applied.') + + +if __name__=='__main__': + main() diff --git a/pilot/agriculture/smoke.py b/pilot/agriculture/smoke.py new file mode 100644 index 0000000..6b73346 --- /dev/null +++ b/pilot/agriculture/smoke.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Native direct-source proof against only the prepared synthetic pilot.""" +import json +import os +from pathlib import Path +import subprocess +import urllib.request +import urllib.error +import uuid + +ROOT=Path('/config/breg') +WORK=ROOT/'smoke' + + +def command(*args,cwd=None): + result=subprocess.run(list(map(str,args)),capture_output=True,text=True,cwd=cwd) + if result.returncode: + raise RuntimeError(f'{args[0]} failed: {result.stderr[-2000:]}') + return result.stdout + + +def token(client): + return command('mint','token','--url','http://127.0.0.1:8091/token','--client-id',client,'--key',ROOT/f'clients/{client}/signing-p256-private-jwk').strip() + + +def request(method,path,bearer,body=None,headers=None): + req=urllib.request.Request('http://127.0.0.1:8090'+path,method=method,data=None if body is None else json.dumps(body).encode(),headers={'Authorization':'Bearer '+bearer,'Accept':'application/json',**({'Content-Type':'application/json'} if body is not None else {}),**(headers or {})}) + try: + with urllib.request.urlopen(req,timeout=15) as response: + return response.status,dict(response.headers),json.load(response) + except urllib.error.HTTPError as error: + return error.code,dict(error.headers),json.load(error) + + +def save(path,body): + path.write_text(json.dumps(body,indent=2)+'\n'); path.chmod(0o600) + + +def main(): + os.umask(0o077) + WORK.mkdir(exist_ok=True,mode=0o700) + service=token('openfn-service'); source=token('evidence-source') + identifier='SYNTHETIC-DIRECT-SMOKE-001' + status,_,found=request('POST','/v1/records/farms:lookup?accessProfile=openfn-service',service,{'selector':'by-local-identifier','values':{'localIdentifier':identifier}}) + if status==404 and found.get('code')=='lookup.unresolved': + status,_,found=request('POST','/v1/records/farms?accessProfile=openfn-service',service,{'data':{'localIdentifier':identifier,'name':'SYNTHETIC-PRIVATE-NAME-CANARY'}}, {'Idempotency-Key':'synthetic-direct-smoke-create-v1'}) + assert status==201,(status,found) + record=found['data']['recordIdentifier'] + else: + assert status==200,(status,found.get('code')) + record=found['data']['recordIdentifier'] + status,headers,_=request('GET',f'/v1/records/farms/{record}?accessProfile=openfn-service',service) + assert status==200 + status,_,problem=request('PATCH',f'/v1/records/farms/{record}?accessProfile=openfn-service',service,{'name':'NEVER'}, {'Idempotency-Key':'synthetic-denied-patch-v1','If-Match':headers.get('ETag',headers.get('Etag',''))}) + assert status==404 and problem.get('code')=='resource.not_found',(status,problem.get('code')) + status,_,problem=request('POST','/v1/records/farms?accessProfile=evidence-source',source,{'data':{'localIdentifier':'SYNTHETIC-DENIED','name':'NEVER'}}, {'Idempotency-Key':'synthetic-denied-create-v1'}) + assert status==404 and problem.get('code')=='resource.not_found',(status,problem.get('code')) + status,_,found=request('POST','/v1/records/farms:lookup?accessProfile=evidence-source',source,{'selector':'by-local-identifier','values':{'localIdentifier':identifier}}) + assert status==200 + assert found['data']['domainData']=={'localIdentifier':identifier} + # Exercise the documented reviewer commands on this separate synthetic record. + status,_,before=request('GET',f'/v1/records/farms/{record}?accessProfile=openfn-service',service) + before_name=before['data']['domainData']['name'] + correction_name='SYNTHETIC-CORRECTED-'+uuid.uuid4().hex[:12] + status,_,draft=request('POST','/v1/records/name-corrections?accessProfile=openfn-service',service,{'data':{'record':record,'name':correction_name,'reason':'Synthetic reviewer command proof','supportingReference':'SYNTHETIC-SMOKE-SUPPORT'}},{'Idempotency-Key':'smoke-correction-'+uuid.uuid4().hex}) + assert status==201,(status,draft.get('code')) + correction=draft['data']['recordIdentifier'] + status,_,draft=request('GET',f'/v1/records/name-corrections/{correction}?accessProfile=openfn-service',service) + submit=next(a for a in draft['data']['request']['actions'] if a['operation']=='submit_request') + status,_,submitted=request('POST',f'/v1/records/name-corrections/{correction}/actions/submit?accessProfile=openfn-service',service,{}, {'If-Match':submit['ifMatch'],'Idempotency-Key':'smoke-submit-'+correction}) + assert status==200,(status,submitted.get('code')) + command('python3','/opt/pilot/agriculture/review.py','inspect',correction) + reviewed=json.loads((ROOT/'review'/f'{correction}.json').read_text()) + approval=next(a for a in reviewed['data']['request']['actions'] if a['operation']=='approve_request') + status,_,denied=request('POST',f'/v1/records/name-corrections/{correction}/actions/stages/review/approve?accessProfile=openfn-service',service,{key:approval[key] for key in ['proposalVersion','effectDigest']},{'If-Match':approval['ifMatch'],'Idempotency-Key':'smoke-denied-approve-'+correction}) + assert status==404 and denied.get('code')=='resource.not_found',(status,denied.get('code')) + command('python3','/opt/pilot/agriculture/review.py','approve',correction) + status,_,after_approval=request('GET',f'/v1/records/farms/{record}?accessProfile=openfn-service',service) + assert after_approval['data']['domainData']['name']==before_name + command('python3','/opt/pilot/agriculture/review.py','inspect',correction) + command('python3','/opt/pilot/agriculture/review.py','apply',correction) + status,_,after_apply=request('GET',f'/v1/records/farms/{record}?accessProfile=openfn-service',service) + assert after_apply['data']['domainData']['name']==correction_name + for suffix,value in [('registered',identifier),('missing','SYNTHETIC-DIRECT-SMOKE-MISSING')]: + name=suffix+'-'+uuid.uuid4().hex[:12] + inputs=WORK/(name+'-subjects.json') + save(inputs,{'subjects':[{'role':'subject','field':'local-identifier','value':value}]}) + command('evidencectl','request','prepare','--profile','/config/evidence-client/profile.json','--requirement','holding-registered','--subjects-file',inputs,'--name',name,cwd=WORK) + retained=WORK/'.evidence/requests'/name + response=retained/'response.json' + status=command('curl','--silent','--show-error','--config',retained/'curl.config','--output',response,'--write-out','%{http_code}',cwd=WORK) + if suffix=='registered': + assert status=='200',status + command('evidencectl','verify',response,'--context',retained/'verification.json','--output',retained/'verified.json') + verified=(retained/'verified.json').read_text() + assert 'SYNTHETIC-PRIVATE-NAME-CANARY' not in verified and identifier not in verified + assert 'urn:example:concept:holding-registered:registered' in verified + else: + problem=json.loads(response.read_text()) + assert problem.get('code')=='evidence.unavailable',(status,problem.get('code')) + save(WORK/'report.json',{'serviceApprovalRefused':True,'directPatchRefused':True,'sourceCreateRefused':True,'sourceNameExcluded':True,'signedRegisteredVerified':True,'missingUnavailable':True,'approvalLeavesFarmUnchanged':True,'manualApplyChangesFarm':True}) + print('PASS: source/read-write ceilings, explicit review/application split, verified registered assertion, and missing unavailable.') + + +if __name__=='__main__': + main() diff --git a/pilot/lightning/admin.exs b/pilot/lightning/admin.exs new file mode 100644 index 0000000..f5dff96 --- /dev/null +++ b/pilot/lightning/admin.exs @@ -0,0 +1,54 @@ +# Run only through the dedicated pilot's release RPC. Uses upstream context +# functions and audits, never direct SQL or a custom Lightning build. +secret_root = System.get_env("PILOT_SECRET_ROOT", "/run/pilot-secrets") +read_json = fn name -> name |> then(&Path.join(secret_root, &1)) |> File.read!() |> Jason.decode!() end +operator = read_json.("operator.json") +user = case Lightning.Accounts.get_user_by_email(operator["email"]) do + nil -> + if Lightning.Accounts.has_one_superuser?(), do: raise("Pilot refuses an existing operator database") + case Lightning.Accounts.register_superuser(operator) do + {:ok, %{user: user}} -> user + {:ok, user} -> user + _ -> raise("Pilot operator setup failed") + end + user -> + if is_nil(Lightning.Accounts.get_user_by_email_and_password(operator["email"], operator["password"])), + do: raise("Pilot operator credentials do not match") + user +end +token_path = Path.join(secret_root, "api-token") +unless File.exists?(token_path) do + token = Lightning.Accounts.generate_api_token(user) + File.write!(token_path, token) + File.chmod!(token_path, 0o600) +end + +config_path = Path.join(secret_root, "project.json") +if File.exists?(config_path) do + config = read_json.("project.json") + project = Lightning.Projects.get_project!(config["projectId"]) + if project.name != "registry-agriculture-pilot", do: raise("Unexpected pilot project") + {:ok, _} = Lightning.Projects.update_project(project, %{ + retention_policy: :retain_all, history_retention_period: 7, dataclip_retention_period: 7 + }, user) + for {kind, trigger_id} <- config["triggerIds"] do + # Submission clients cannot authenticate directly to the trusted event path. + {name, key_file} = if kind == "committed", + do: {"Pilot committed events", "committed-api-key"}, + else: {"Pilot incoming events", "webhook-api-key"} + auth = case Enum.find(Lightning.WebhookAuthMethods.list_for_project(project), &(&1.name == name)) do + nil -> + {:ok, method} = Lightning.WebhookAuthMethods.create_auth_method(%{ + name: name, auth_type: :api, project_id: project.id, + api_key: File.read!(Path.join(secret_root, key_file)) |> String.trim() + }, actor: user) + method + method -> method + end + trigger = Lightning.Workflows.get_webhook_trigger(trigger_id) + if is_nil(trigger), do: raise("Pilot webhook not found") + {:ok, _} = Lightning.WebhookAuthMethods.update_trigger_auth_methods(trigger, [auth], actor: user) + end +end +IO.puts("Pilot operator and configured webhook protections are ready.") +:ok diff --git a/pilot/lightning/project.mjs b/pilot/lightning/project.mjs new file mode 100644 index 0000000..38ba93d --- /dev/null +++ b/pilot/lightning/project.mjs @@ -0,0 +1,148 @@ +import { createHash } from 'node:crypto'; + +// Stable identities make provisioning repeatable. This project is dedicated to the pilot. +export function id(name) { + const bytes = createHash('sha256').update(`registry-openfn-agriculture-v1:${name}`).digest(); + bytes[6] = (bytes[6] & 15) | 80; + bytes[8] = (bytes[8] & 63) | 128; + const hex = bytes.subarray(0, 16).toString('hex'); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; +} + +export const projectId = id('project'); +export const triggerIds = Object.fromEntries(['registration', 'correction', 'committed'].map(name => [name, id(`${name}:trigger`)])); + +const checked = ` +fn(state => { + const result = state.data.result; + if (result.branch !== 'succeeded') { + const error = new Error('Registry operation did not complete: ' + result.branch); + error.name = 'RegistryWorkflowError'; + throw error; + } + return state; +});`; + +export function projectDocument(bindings, credentials = {}, enabled = false) { + const literal = value => JSON.stringify(value).replace(/\u2028/g, "\\u2028").replace(/\u2029/g, "\\u2029"); + const profile = literal(bindings.accessProfile); + const validateSubmission = `fn(state => { + const value = state.data.submissionId; + if (typeof value !== 'string' || !/^[A-Za-z0-9_-]{1,100}$/.test(value)) throw new Error('A stable submissionId is required'); + return state; + });`; + const registration = `${validateSubmission} +createRecord(state => ({ + operationIdentifier: ${literal(bindings.createFarmOperation)}, accessProfile: ${profile}, + data: {localIdentifier: state.data.localIdentifier, name: state.data.name}, + idempotencyKey: 'registration:' + state.data.submissionId, as: 'result' +})); +${checked} +fn(state => ({data: {status: 'registered', recordId: state.data.result.value.data.recordIdentifier, traceId: state.data.result.traceId}}));`; + const correction = `${validateSubmission} +createChangeRequest(state => ({ + operationIdentifier: ${literal(bindings.createCorrectionOperation)}, accessProfile: ${profile}, + entityIdentifier: 'name-correction', route: 'name-corrections', + data: {record: state.data.recordId, name: state.data.name, reason: state.data.reason, supportingReference: state.data.supportingReference}, + idempotencyKey: 'correction:' + state.data.submissionId, + as: 'result' +})); +${checked} +fn(state => ({...state, data: {...state.data, requestId: state.data.result.value.data.recordIdentifier}})); +getRecord(state => ({route: 'name-corrections', recordIdentifier: state.data.requestId, accessProfile: ${profile}, as: 'result'})); +${checked} +fn(async state => { + const current = state.data.result.value.data; + const expected = {record: state.data.recordId, name: state.data.name, reason: state.data.reason, supportingReference: state.data.supportingReference}; + if (Object.entries(expected).some(([field, value]) => typeof value !== 'string' || current.domainData[field] !== value)) { + throw new Error('Correction content does not match the submitted event'); + } + const request = current.request; + const action = request?.actions?.find(action => action.operation === 'submit_request'); + if (action) { + state = await executeLifecycleAction({ + entityIdentifier: 'name-correction', route: 'name-corrections', + recordIdentifier: state.data.requestId, accessProfile: ${profile}, + operation: 'submit_request', ifMatch: action.ifMatch, + idempotencyKey: 'correction-submit:' + state.data.submissionId, as: 'result' + })(state); + if (state.data.result.branch !== 'succeeded') throw new Error('Correction submission did not complete: ' + state.data.result.branch); + } else if (!['submitted', 'approved', 'applied', 'rejected'].includes(request?.bregState)) { + throw new Error('Correction is not ready for submission'); + } + return {data: {status: action ? 'submitted' : 'already_submitted', requestId: state.data.requestId}}; +});`; + const evidence = ` +fn(state => { + const event = state.data.event; + const committed = state.data.data; + const delivery = state.data.delivery; + const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; + if (!event || typeof event.source !== 'string' || !event.source.startsWith('urn:registrystack:registry:') || event.source.length > 512 || + typeof event.id !== 'string' || !uuid.test(event.id) || + !committed || typeof committed.recordId !== 'string' || !uuid.test(committed.recordId) || + !Number.isSafeInteger(committed.revision) || committed.revision < 1 || + !Number.isSafeInteger(delivery?.generation) || delivery.generation < 1 || + typeof committed.values?.['local-identifier'] !== 'string' || !committed.values['local-identifier']) { + throw new Error('A committed registry event with identity, revision, and selector is required'); + } + return state; +}); +requestEvidence({ + requirement: ${literal(bindings.requirement)}, + selectors: {'local-identifier': {valueFrom: 'data.values.local-identifier'}}, as: 'result' +}); +fn(state => { + const result = state.data.result; + if (result.branch !== 'succeeded') throw new Error('Evidence request did not complete: ' + result.branch); + if (selectSupportedValue(result.assertion, ${literal(bindings.registeredConcept)}) !== true) throw new Error('A verified registration assertion is required'); + // Revision identifies the committed event for destination ordering. The + // assertion verifies current registration; it does not attest a historical + // snapshot at that event revision. Never replace this with evidence metadata. + return {data: { + source: state.data.event.source, eventId: state.data.event.id, effect: 'record-sync', + recordId: state.data.data.recordId, revision: state.data.data.revision, + values: {registered: true} + }, integration: { + eventSource: state.data.event.source, eventId: state.data.event.id, + committedRevision: state.data.data.revision, deliveryGeneration: state.data.delivery.generation, + evidenceTraceId: result.trace_id ?? null, evidenceObservedAt: result.assertion.observedAt + }}; +});`; + const destination = `post('/updates', state => state.data, { + headers: state => ({'x-destination-key': state.configuration.apiKey}), + maxRedirections: 0, retries: 0, timeout: 3000 +}); +fn(state => ({data: {status: state.data.status, revision: state.data.revision}, integration: state.integration}));`; + + function workflow(name, steps) { + const jobs = steps.map(([key, adaptor, body, credential]) => ({ + id: id(`${name}:${key}`), name: key, adaptor, body, + project_credential_id: credentials[credential] ?? null, + })); + return { + id: id(name), name: `Agriculture ${name}`, + jobs, + triggers: [{id: triggerIds[name], type: 'webhook', enabled, has_auth_method: true, + webhook_reply: name === 'committed' ? 'before_start' : 'after_completion', + ...(name === 'committed' ? {} : {webhook_response_config: {success_code: 200, error_code: 422}}), + }], + edges: jobs.map((job, index) => ({ + id: id(`${name}:edge:${index}`), enabled: true, target_job_id: job.id, + ...(index === 0 ? {source_trigger_id: triggerIds[name], condition_type: 'always'} : {source_job_id: jobs[index - 1].id, condition_type: 'on_job_success'}), + })), + }; + } + return { + id: projectId, name: 'registry-agriculture-pilot', + description: 'Synthetic governed registration, correction, and signed registration evidence.', + workflows: [ + workflow('registration', [['register', '@openfn/language-registry-breg@local', registration, 'breg']]), + workflow('correction', [['submit-correction', '@openfn/language-registry-breg@local', correction, 'breg']]), + workflow('committed', [ + ['verify-registration', '@openfn/language-registry-evidence@local', evidence, 'evidence'], + ['record-update', '@openfn/language-http@local', destination, 'destination'], + ]), + ], + }; +} diff --git a/pilot/lightning/provision.mjs b/pilot/lightning/provision.mjs new file mode 100644 index 0000000..1124024 --- /dev/null +++ b/pilot/lightning/provision.mjs @@ -0,0 +1,242 @@ +import { mkdir, readFile, open, rename, link, unlink } from 'node:fs/promises'; +import { resolve, join } from 'node:path'; +import { randomBytes, generateKeyPairSync, createPublicKey, randomUUID } from 'node:crypto'; +import { fileURLToPath } from 'node:url'; +import { projectDocument, projectId, triggerIds } from './project.mjs'; + +// Publish complete owner-only bytes. A failed write never truncates a checkpoint. +export async function privateFile(path, value, exclusive = false) { + const temporary = `${path}.${randomUUID()}.tmp`; + let handle; + try { + handle = await open(temporary, 'wx', 0o600); + await handle.writeFile(value); + await handle.sync(); + await handle.close(); + handle = undefined; + if (exclusive) await link(temporary, path); + else await rename(temporary, path); + } finally { + await handle?.close(); + await unlink(temporary).catch(error => { if (error.code !== 'ENOENT') throw error; }); + } +} + +async function optionalText(path) { + try { return await readFile(path, 'utf8'); } + catch (error) { if (error.code !== 'ENOENT') throw error; return undefined; } +} + +function envFields(text) { + return Object.fromEntries(text.trim().split('\n').map(line => { + const separator = line.indexOf('='); + if (separator < 1) throw new Error('Existing pilot environment file is invalid'); + return [line.slice(0, separator), line.slice(separator + 1)]; + })); +} + +async function prepareEnvironment(files, secrets) { + const checkpoint = join(secrets, 'environment.json'); + let saved = await optionalText(checkpoint); + if (saved === undefined) { + const lightning = await optionalText(join(files, 'lightning.env')); + const postgres = await optionalText(join(files, 'postgres.env')); + const worker = await optionalText(join(files, 'worker.env')); + if (!lightning && worker) throw new Error('Existing worker keys have no matching Lightning environment; restore lightning.env before preparing'); + let lightningFields; + let databasePassword; + let keys; + let shared; + if (lightning) { + lightningFields = envFields(lightning); + const url = new URL(lightningFields.DATABASE_URL); + if (url.hostname !== 'lightning-db' || url.username !== 'lightning' || url.pathname !== '/lightning') { + throw new Error('Existing Lightning database binding does not match this pilot'); + } + databasePassword = decodeURIComponent(url.password); + shared = lightningFields.WORKER_SECRET; + if (!databasePassword || !shared || !lightningFields.WORKER_RUNS_PRIVATE_KEY) { + throw new Error('Existing Lightning environment is incomplete; restore it before preparing'); + } + const privateKey = Buffer.from(lightningFields.WORKER_RUNS_PRIVATE_KEY, 'base64'); + keys = {publicKey: createPublicKey(privateKey).export({type: 'spki', format: 'pem'})}; + } else { + databasePassword = postgres ? envFields(postgres).POSTGRES_PASSWORD : randomBytes(32).toString('hex'); + if (!databasePassword || !/^[a-zA-Z0-9_-]+$/.test(databasePassword)) throw new Error('Existing pilot database password cannot be safely retained'); + shared = randomBytes(32).toString('hex'); + keys = generateKeyPairSync('rsa', {modulusLength: 2048, + privateKeyEncoding: {type: 'pkcs8', format: 'pem'}, publicKeyEncoding: {type: 'spki', format: 'pem'}}); + } + if (postgres && envFields(postgres).POSTGRES_PASSWORD !== databasePassword) { + throw new Error('Existing pilot database environments disagree; no credentials were replaced'); + } + const publicKey = Buffer.from(keys.publicKey).toString('base64'); + if (worker) { + const fields = envFields(worker); + if (fields.WORKER_SECRET !== shared || fields.WORKER_LIGHTNING_PUBLIC_KEY !== publicKey) { + throw new Error('Existing pilot worker keys disagree; no keys were replaced'); + } + } + const environment = { + 'postgres.env': postgres ?? `POSTGRES_USER=lightning\nPOSTGRES_DB=lightning\nPOSTGRES_PASSWORD=${databasePassword}\n`, + 'lightning.env': lightning ?? [ + `DATABASE_URL=postgresql://lightning:${databasePassword}@lightning-db:5432/lightning`, + `SECRET_KEY_BASE=${randomBytes(64).toString('base64')}`, + `PRIMARY_ENCRYPTION_KEY=${randomBytes(32).toString('base64')}`, + `WORKER_RUNS_PRIVATE_KEY=${Buffer.from(keys.privateKey).toString('base64')}`, + `WORKER_SECRET=${shared}`, 'PHX_SERVER=true', 'LISTEN_ADDRESS=0.0.0.0', 'MAX_DATACLIP_SIZE_MB=1', 'DISABLE_DB_SSL=true', + 'ORIGINS=http://localhost:4000,http://127.0.0.1:4000,http://lightning:4000', + 'WORKER_MAX_RUN_DURATION_SECONDS=60', 'USAGE_TRACKING_ENABLED=false', + ].join('\n') + '\n', + 'worker.env': worker ?? [ + `WORKER_LIGHTNING_PUBLIC_KEY=${publicKey}`, `WORKER_SECRET=${shared}`, + 'WORKER_CAPACITY=2', 'NODE_OPTIONS=--experimental-vm-modules', + ].join('\n') + '\n', + }; + try { await privateFile(checkpoint, JSON.stringify(environment), true); } + catch (error) { if (error.code !== 'EEXIST') throw error; } + saved = await readFile(checkpoint, 'utf8'); + } + const environment = JSON.parse(saved); + for (const name of ['postgres.env', 'lightning.env', 'worker.env']) { + if (typeof environment[name] !== 'string') throw new Error('Pilot environment checkpoint is incomplete'); + const path = join(files, name); + const current = await optionalText(path); + if (current !== undefined && current !== environment[name]) { + throw new Error('Existing pilot environment differs from its retained checkpoint; no keys were replaced'); + } + if (current === undefined) { + try { await privateFile(path, environment[name], true); } + catch (error) { if (error.code !== 'EEXIST') throw error; } + } + } +} + +export async function run(command = 'prepare', runtime = 'pilot/agriculture/.runtime', { + base = process.env.OPENFN_URL ?? 'http://127.0.0.1:4000', fetchImpl = fetch, +} = {}) { + runtime = resolve(runtime); + const secrets = join(runtime, 'openfn', 'secrets'); + await mkdir(secrets, {recursive: true, mode: 0o700}); + const json = async path => JSON.parse(await readFile(path, 'utf8')); + const fresh = async (name, value) => { + try { await privateFile(join(secrets, name), value, true); } + catch (error) { if (error.code !== 'EEXIST') throw error; } + }; + if (command === 'prepare') { + await fresh('operator.json', JSON.stringify({email: 'operator@agriculture.example.invalid', + first_name: 'Pilot', last_name: 'Operator', password: randomBytes(32).toString('base64url')})); + await fresh('webhook-api-key', randomBytes(32).toString('hex')); + await fresh('committed-api-key', randomBytes(32).toString('hex')); + await fresh('destination-api-key', randomBytes(32).toString('hex')); + const files = join(runtime, 'openfn'); + await prepareEnvironment(files, secrets); + await mkdir(join(runtime, 'destination'), {recursive: true, mode: 0o700}); + await privateFile(join(runtime, 'destination', 'api-key'), await readFile(join(secrets, 'destination-api-key'))); + await privateFile(join(runtime, 'bridge', 'openfn-api-key'), await readFile(join(secrets, 'committed-api-key'))); + const bridgeSource = (await readFile(join(runtime, 'bridge', 'expected-source'), 'utf8')).trim(); + await privateFile(join(files, 'bridge.env'), `BREG_EXPECTED_SOURCE=${bridgeSource}\nOPENFN_WEBHOOK_URL=http://lightning:4000/i/${triggerIds.committed}\n`); + console.log('Pilot-local operator and webhook secret files prepared.'); + return; + } + const token = (await readFile(join(secrets, 'api-token'), 'utf8')).trim(); + async function api(path, method = 'GET', body) { + const response = await fetchImpl(new URL(path, base), {method, redirect: 'error', signal: AbortSignal.timeout(15000), + headers: {authorization: `Bearer ${token}`, 'content-type': 'application/json'}, + ...(body === undefined ? {} : {body: JSON.stringify(body)})}); + if (!response.ok) throw new Error(`OpenFn ${method} ${path} returned HTTP ${response.status}`); + return response.json(); + } + const generated = await json(join(runtime, 'workflow-bindings.json')); + const bindings = { + accessProfile: generated.breg.accessProfile, + createFarmOperation: generated.breg.farmCreateOperation, + createCorrectionOperation: generated.breg.correctionCreateOperation, + requirement: generated.evidence.requirement, + registeredConcept: generated.evidence.registeredConcept, + }; + if (command === 'provision') { + const empty = projectDocument(bindings); + // Create the dedicated project before associating credentials; jobs stay disabled. + const probe = await fetchImpl(new URL(`/api/provision/${projectId}`, base), { + redirect: 'error', signal: AbortSignal.timeout(5000), headers: {authorization: `Bearer ${token}`}, + }); + if (probe.status === 404) await api('/api/provision', 'POST', {...empty, workflows: []}); + else if (!probe.ok) throw new Error(`Cannot inspect pilot project: HTTP ${probe.status}`); + let state; + try { state = await json(join(secrets, 'credentials.json')); } catch (error) { if (error.code !== 'ENOENT') throw error; state = {}; } + const credentialBodies = { + breg: await json(join(runtime, 'openfn', 'breg-credential.json')), + evidence: await json(join(runtime, 'openfn', 'evidence-credential.json')), + destination: {baseUrl: 'http://destination:8082', apiKey: (await readFile(join(secrets, 'destination-api-key'), 'utf8')).trim()}, + }; + const statePath = join(secrets, 'credentials.json'); + if (state.credentialFormat !== 2) { + // Legacy body layouts are not inspectable through the metadata API. Keep + // their associations untouched and select a fresh, checkpointed namespace. + state = {credentialFormat: 2, previous: state, credentialNames: {}}; + await privateFile(statePath, JSON.stringify(state)); + } + state.credentialNames ??= {}; + for (const [name, body] of Object.entries(credentialBodies)) { + if (state[name]) continue; + if (!state.credentialNames[name]) { + state.credentialNames[name] = `Agriculture ${name} main ${randomUUID()}`; + await privateFile(statePath, JSON.stringify(state)); + } + // Recover a POST which committed but whose response/checkpoint was lost. + // This pinned API returns metadata only, never credential body values. + const listed = await api(`/api/credentials?project_id=${projectId}`); + const matches = listed.credentials.filter(value => value.name === state.credentialNames[name]); + if (matches.length > 1) throw new Error('Pilot credential metadata is ambiguous; no credentials were replaced'); + let credential = matches[0]; + if (!credential) { + const result = await api('/api/credentials', 'POST', {name: state.credentialNames[name], schema: 'raw', + credential_bodies: [{name: 'main', body}], project_credentials: [{project_id: projectId}]}); + credential = result.credential; + } + const association = credential.schema === 'raw' && credential.project_credentials.find(value => value.project_id === projectId); + if (!association) throw new Error('OpenFn did not associate the intended raw credential with the pilot'); + state[name] = association.id; + await privateFile(statePath, JSON.stringify(state)); + } + await api('/api/provision', 'POST', projectDocument(bindings, state)); + await privateFile(join(secrets, 'project.json'), JSON.stringify({projectId, triggerIds})); + console.log('Pilot workflows provisioned with disabled triggers. Attach webhook authentication before enabling.'); + } else if (command === 'enable') { + const intakeKey = (await readFile(join(secrets, 'webhook-api-key'), 'utf8')).trim(); + const committedKey = (await readFile(join(secrets, 'committed-api-key'), 'utf8')).trim(); + if (!intakeKey || !committedKey || intakeKey === committedKey) { + throw new Error('Refusing to enable webhooks without distinct intake and committed-event keys'); + } + // Lightning v2.18.2 authenticates GET /i/* through WebhookAuth, then + // WebhooksController.check returns 200 without creating a work order. + // Missing credentials return 401; wrong credentials conceal the route as + // 404. These checks work while triggers are disabled, unlike POST intake. + async function check(trigger, key) { + return fetchImpl(new URL(`/i/${trigger}`, base), {method: 'GET', redirect: 'error', + signal: AbortSignal.timeout(5000), headers: {accept: 'application/json', + ...(key === undefined ? {} : {'x-api-key': key})}}); + } + for (const [kind, trigger] of Object.entries(triggerIds)) { + if ((await check(trigger)).status !== 401) { + throw new Error('Refusing to enable a webhook without authentication'); + } + const ownKey = kind === 'committed' ? committedKey : intakeKey; + const otherKey = kind === 'committed' ? intakeKey : committedKey; + if ((await check(trigger, otherKey)).status !== 404) { + throw new Error('Refusing to enable webhooks whose intake and committed-event authority overlap'); + } + if ((await check(trigger, ownKey)).status !== 200) { + throw new Error('Refusing to enable a webhook whose intended key is not accepted'); + } + } + const state = await json(join(secrets, 'credentials.json')); + await api('/api/provision', 'POST', projectDocument(bindings, state, true)); + console.log('Pilot authenticated triggers enabled.'); + } else throw new Error('Expected prepare, provision, or enable'); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + run(process.argv[2], process.argv[3]).catch(error => { console.error(error.message); process.exitCode = 1; }); +} diff --git a/pilot/lightning/recovery.exs b/pilot/lightning/recovery.exs new file mode 100644 index 0000000..f58cd88 --- /dev/null +++ b/pilot/lightning/recovery.exs @@ -0,0 +1,66 @@ +# Bounded operator inspection/retry for this pilot only. Invoked through the +# unmodified v2.18.2 release RPC. WorkOrders.retry is the owning context used by +# WorkflowController.retry_run; no direct SQL mutations or replacement dataclips. +import Ecto.Query + +request = pilot_recovery_request |> Jason.decode!() +secret_root = System.get_env("PILOT_SECRET_ROOT", "/run/pilot-secrets") +config = Path.join(secret_root, "project.json") |> File.read!() |> Jason.decode!() +project = Lightning.Projects.get_project!(config["projectId"]) +if project.name != "registry-agriculture-pilot", do: raise("Unexpected pilot project") +trigger = Lightning.Workflows.get_webhook_trigger(config["triggerIds"]["committed"]) +if is_nil(trigger), do: raise("Pilot committed trigger is unavailable") +workflow = Lightning.Workflows.get_workflow!(trigger.workflow_id) +if workflow.project_id != project.id, do: raise("Unexpected pilot workflow") +record_id = request["recordId"] +if not is_binary(record_id) or not Regex.match?(~r/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/, record_id), + do: raise("Expected a synthetic record identity") + +identities = Lightning.Repo.all(from order in Lightning.WorkOrder, + join: dataclip in assoc(order, :dataclip), + where: order.workflow_id == ^workflow.id and + fragment("?->'data'->>'recordId'", dataclip.body) == ^record_id, + order_by: [asc: order.inserted_at], limit: 20, + select: {order.id, fragment("?->'event'->>'id'", dataclip.body), + fragment("?->'delivery'->'generation'", dataclip.body)}) +# Dataclip.body is deliberately excluded from ordinary Ecto loads. Select only +# the two identity members needed here, without loading or logging the payload. +orders = Enum.map(identities, fn {id, _, _} -> + Lightning.WorkOrders.get(id, include: [runs: [steps: :job]]) +end) +identity_by_order = Map.new(identities, fn {id, event, generation} -> {id, {event, generation}} end) + +terminal_failures = [:failed, :crashed, :killed, :cancelled, :exception, :lost] +result = case request["action"] do + "inspect" -> + %{workOrders: Enum.map(orders, fn order -> + %{id: order.id, state: order.state, eventId: elem(identity_by_order[order.id], 0), + generation: elem(identity_by_order[order.id], 1), + runs: order.runs |> Enum.sort_by(& &1.inserted_at, DateTime) |> Enum.map(fn run -> + %{id: run.id, state: run.state, queue: run.queue, steps: Enum.map(run.steps, fn step -> + %{id: step.id, job: step.job.name, exitReason: step.exit_reason} + end)} + end)} + end)} + "retry" -> + order = Enum.find(orders, &(&1.id == request["workOrderId"])) + if is_nil(order) or order.state not in terminal_failures, + do: raise("Expected one failed pilot work order") + run = Enum.max_by(order.runs, & &1.inserted_at, DateTime) + if run.id != request["expectedRunId"] or run.state not in terminal_failures, + do: raise("Pilot run changed before retry") + failed_steps = Enum.filter(run.steps, &(&1.exit_reason in ["fail", "crash", "cancel", "kill", "exception", "lost"])) + if length(failed_steps) != 1, do: raise("Expected one failed retained step") + step = hd(failed_steps) + if step.job.name not in ["verify-registration", "record-update"], + do: raise("Unexpected failed pilot step") + operator = Path.join(secret_root, "operator.json") |> File.read!() |> Jason.decode!() + user = Lightning.Accounts.get_user_by_email(operator["email"]) + if is_nil(user), do: raise("Pilot operator is unavailable") + :ok = Lightning.WorkOrders.limit_run_creation(project.id) + {:ok, retry} = Lightning.WorkOrders.retry(run.id, step.id, created_by: user) + %{workOrderId: order.id, previousRunId: run.id, runId: retry.id, retriedJob: step.job.name} + _ -> raise("Unsupported pilot recovery operation") +end +IO.puts("PILOT_RECOVERY_JSON:" <> Jason.encode!(result)) +:ok diff --git a/pilot/recovery.py b/pilot/recovery.py new file mode 100644 index 0000000..2b8c74a --- /dev/null +++ b/pilot/recovery.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python3 +"""Bounded live failure/recovery acceptance for the dedicated synthetic pilot. + +Run --retry-smoke after repairing a failed accepted smoke work order. +Run --replay-smoke after its destination effects are verified, if BREG retained +the same event as a dead letter because its earlier acknowledgement was lost. +Run --scenario before-accept|after-accept|pending-restart|all only while the +dedicated pilot is idle. These scenarios briefly stop their named services, +create three separate retained synthetic identities, and restore service +availability in finally blocks. No volume, record, dataclip, or event is erased. +""" +import argparse +import base64 +import json +import os +import subprocess +import sys +import time +import uuid +from pathlib import Path + +from smoke import (COMPOSE, REPO, RUNTIME, SAFE_ID, Pilot, SmokeFailure, captured, + http_json, json_output, operation_path, private_write, read_secret, require) + +FAILURES = {'failed', 'crashed', 'killed', 'cancelled', 'exception', 'lost'} +SUCCESS = {'success', 'completed'} + + +def rpc(record_id, action='inspect', **arguments): + require(isinstance(record_id, str) and bool(SAFE_ID.fullmatch(record_id)), 'Invalid recovery record identity') + document = json.dumps({'action': action, 'recordId': record_id, **arguments}, separators=(',', ':')) + encoded = base64.urlsafe_b64encode(document.encode()).decode('ascii') + expression = 'Code.eval_string(File.read!("/opt/pilot/lightning/recovery.exs"), [pilot_recovery_request: Base.url_decode64!("' + encoded + '")], file: "/opt/pilot/lightning/recovery.exs"); :ok' + output = captured(COMPOSE + ['exec', '-T', 'lightning', '/app/bin/lightning', 'rpc', expression], + 'Scoped OpenFn recovery ' + action) + lines = [line.removeprefix('PILOT_RECOVERY_JSON:') for line in output.splitlines() if line.startswith('PILOT_RECOVERY_JSON:')] + require(len(lines) == 1, 'Scoped OpenFn recovery did not return one report') + return json_output(lines[0], 'Scoped OpenFn recovery') + + +def database(query): + # SQL is authored here and contains only validated UUIDs. Credentials remain + # in the existing database container environment, and output stays captured. + command = COMPOSE + ['exec', '-T', 'breg-db', 'sh', '-c', + 'exec psql -X -A -t -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d agriculture'] + try: + result = subprocess.run(command, cwd=REPO, input='BEGIN READ ONLY;\n' + query + '\nROLLBACK;\n', + capture_output=True, text=True, timeout=30) + except (OSError, subprocess.TimeoutExpired): + raise SmokeFailure('Read-only delivery inspection did not complete') from None + require(result.returncode == 0, 'Read-only delivery inspection failed; output withheld') + lines = [line for line in result.stdout.splitlines() if line and line not in ['BEGIN', 'ROLLBACK']] + require(len(lines) == 1, 'Delivery inspection returned an unexpected shape') + return json_output(lines[0], 'Delivery inspection') + + +def event_for_record(record_id): + require(bool(SAFE_ID.fullmatch(record_id)), 'Invalid recovery record identity') + return database("SELECT coalesce(json_agg(event_id), '[]'::json) FROM registry_internal.registry_outbox " + "WHERE entity_id='farm' AND payload IS NOT NULL AND " + "convert_from(payload,'UTF8')::jsonb->>'recordId'='" + record_id + "';") + + +def delivery(event_id): + require(isinstance(event_id, str) and bool(SAFE_ID.fullmatch(event_id)), 'Invalid recovery event identity') + rows = database("SELECT coalesce(json_agg(json_build_object('eventId',s.event_id,'deliveryId',s.compiled_delivery_id," + "'generation',s.generation,'state',s.state,'attempt',s.attempt,'payloadAvailable',o.payload IS NOT NULL)), '[]'::json) " + "FROM registry_internal.registry_webhook_delivery_state s JOIN registry_internal.registry_outbox o USING(event_id) " + "WHERE s.event_id='" + event_id + "';") + require(isinstance(rows, list) and len(rows) == 1, 'Expected one scoped BREG delivery') + return rows[0] + + +class Recovery: + def __init__(self, timeout=180): + self.timeout = timeout + self.pilot = Pilot(timeout) + self.path = RUNTIME / 'recovery-state.json' + if self.path.exists(): + self.state = json_output(read_secret(self.path), 'Retained recovery state') + require(self.state.get('schema') == 'registry-openfn-recovery/v1', 'Unsupported recovery state') + else: + self.state = {'schema': 'registry-openfn-recovery/v1', 'runId': uuid.uuid4().hex[:16], 'scenarios': {}} + self.save() + + def save(self): + private_write(self.path, self.state) + + def poll(self, function, condition, label): + end = time.monotonic() + self.timeout + while time.monotonic() < end: + try: + # Only read-only observations are passed here. Service restart + # can interrupt one RPC without invalidating accepted work. + value = function() + except SmokeFailure: + time.sleep(2) + continue + if condition(value): + return value + time.sleep(2) + raise SmokeFailure(label + ' did not complete within the deadline') + + def service(self, command, *services): + allowed = {'lightning', 'bridge', 'breg', 'worker', 'destination'} + require(command in {'stop', 'start', 'restart'} and set(services) <= allowed, 'Unsupported recovery service operation') + captured(COMPOSE + [command, *services], 'Pilot recovery service ' + command, timeout=180) + + def ready(self): + def probe(): + try: + return http_json('http://127.0.0.1:4000/health_check', timeout=5, parse_json=False)[0] + except SmokeFailure: + return None + self.poll(probe, lambda status: status == 200, 'OpenFn readiness') + + def destination_ready(self): + def probe(): + try: + return self.pilot.destination() + except SmokeFailure: + return None + self.poll(probe, lambda status: isinstance(status, dict), 'Destination readiness') + + def registry_ready(self): + def probe(): + try: + return self.pilot.metadata() + except SmokeFailure: + return None + self.poll(probe, lambda value: isinstance(value, dict), 'Registry readiness') + + def scenario(self, name): + if name not in self.state['scenarios']: + self.state['scenarios'][name] = {'phase': 'prepared', 'submissionId': 'recovery-' + name + '-' + self.state['runId'], + 'localIdentifier': 'SYNTHETIC-RECOVERY-' + name.upper() + '-' + self.state['runId']} + self.save() + return self.state['scenarios'][name] + + def create(self, scenario): + metadata = self.pilot.metadata() + operation = self.pilot.select(metadata, 'farm', 'create') + body = {'data': {'localIdentifier': scenario['localIdentifier'], 'name': 'Synthetic Recovery Holding'}} + response = self.pilot.breg(operation_path(operation), 'POST', body, + {'Idempotency-Key': 'recovery:' + scenario['submissionId']}) + record_id = response.get('data', {}).get('recordIdentifier') + require(isinstance(record_id, str) and bool(SAFE_ID.fullmatch(record_id)), 'Recovery create returned no record identity') + require('recordId' not in scenario or scenario['recordId'] == record_id, 'Retained recovery record changed') + scenario['recordId'] = record_id + self.save() + + def orders(self, record_id): + return rpc(record_id).get('workOrders', []) + + def accepted(self, scenario): + orders = self.poll(lambda: self.orders(scenario['recordId']), lambda items: len(items) >= 1, 'OpenFn work-order acceptance') + require(len({order['eventId'] for order in orders}) == 1, 'Unexpected recovery event multiplicity') + scenario['eventId'] = orders[0]['eventId'] + self.save() + observed = self.poll(lambda: delivery(scenario['eventId']), lambda item: item['state'] == 'delivered', 'BREG acceptance acknowledgement') + require(observed['payloadAvailable'] is False, 'Delivered BREG event retained its payload unexpectedly') + return orders + + def effects(self, record_id): + original = self.pilot.state.get('recordId') + self.pilot.state['recordId'] = record_id + try: + return self.pilot.destination_rows() + finally: + if original is None: + self.pilot.state.pop('recordId', None) + else: + self.pilot.state['recordId'] = original + + def verify_effect(self, scenario): + rows = self.poll(lambda: self.effects(scenario['recordId']), lambda value: len(value.get('effects', [])) == 1, + 'Recovered destination effect') + require(len(rows.get('records', [])) == 1 and rows['records'][0]['registered'] == 1 and + rows['records'][0]['revision'] == 1 and rows['effects'][0]['outcome'] == 'applied' and + rows['effects'][0]['event_id'] == scenario['eventId'], 'Recovered effect differs from the retained event') + self.poll(lambda: self.orders(scenario['recordId']), + lambda orders: any(order['state'] in SUCCESS for order in orders), 'Recovered OpenFn completion') + + def retry_failed(self, record_id): + orders = self.orders(record_id) + require(bool(orders), 'No retained committed work order matches the record') + results = [] + for order in orders: + if order['state'] in FAILURES: + require(bool(order.get('runs')), 'Failed work order has no retained run') + results.append(rpc(record_id, 'retry', workOrderId=order['id'], expectedRunId=order['runs'][-1]['id'])) + self.poll(lambda: self.orders(record_id), lambda items: all(order['state'] in SUCCESS for order in items), + 'Retried OpenFn work order') + return results + + def replay_smoke(self): + record_id = self.pilot.state['recordId'] + orders = self.orders(record_id) + require(bool(orders) and all(order['state'] in SUCCESS for order in orders), + 'Complete retained OpenFn retries before replaying its BREG event') + before = self.effects(record_id) + expected_events = {order['eventId'] for order in orders} + require({row['event_id'] for row in before['effects']} == expected_events, + 'Verify destination effects before replaying BREG dead letters') + replayed = 0 + for event_id in sorted(expected_events): + observed = delivery(event_id) + if observed['state'] == 'delivered': + continue + require(observed['state'] == 'dead_lettered' and observed['payloadAvailable'], + 'Smoke event is not a retained replayable dead letter') + require(observed['deliveryId'] in ['events.farm.farm-created-v1.webhook', 'events.farm.farm-patched-v1.webhook'], + 'Unexpected compiled smoke delivery') + captured(COMPOSE + ['run', '--rm', '--no-deps', 'tools', 'bregctl', '--format', 'json', 'webhook', 'replay', + '--runtime-config', '/config/breg/runtime.json', '--event-id', event_id, + '--delivery-id', observed['deliveryId'], '--expected-generation', str(observed['generation'])], + 'Explicit retained smoke BREG replay') + settled = self.poll(lambda: delivery(event_id), lambda value: value['state'] == 'delivered', + 'Replayed smoke acknowledgement') + require(settled['generation'] == observed['generation'] + 1 and settled['payloadAvailable'] is False, + 'Replayed smoke event did not retain its identity and advance one generation') + replayed += 1 + self.poll(lambda: self.orders(record_id), lambda items: bool(items) and all(order['state'] in SUCCESS for order in items), + 'Replayed smoke work-order completion') + after = self.effects(record_id) + require(before == after, 'Replayed smoke event changed an already committed destination effect') + self.state['smokeReplay'] = {'status': 'passed', 'recordId': record_id, 'events': sorted(expected_events)} + self.save() + return replayed + + def before_accept(self): + scenario = self.scenario('before-accept') + if scenario['phase'] == 'complete': + return self.verify_effect(scenario) + if 'eventId' not in scenario: + self.service('stop', 'lightning') + try: + self.create(scenario) + events = event_for_record(scenario['recordId']) + require(len(events) == 1, 'Expected one retained failure event') + scenario['eventId'] = events[0] + scenario['phase'] = 'forwarding-failure' + self.save() + observed = self.poll(lambda: delivery(scenario['eventId']), lambda value: value['state'] == 'dead_lettered', 'BREG dead letter') + require(observed['payloadAvailable'] and observed['attempt'] == 5, 'Expected exhausted retained BREG delivery') + scenario['deadLetterGeneration'] = observed['generation'] + self.save() + finally: + self.service('start', 'lightning') + self.ready() + observed = delivery(scenario['eventId']) + require('deadLetterGeneration' in scenario, 'Interrupted failure injection did not prove a dead letter; retained identity was preserved') + if observed['state'] == 'dead_lettered': + require(observed['payloadAvailable'], 'Dead letter payload expired before replay') + require(observed['deliveryId'] == 'events.farm.farm-created-v1.webhook', 'Unexpected compiled recovery delivery') + captured(COMPOSE + ['run', '--rm', '--no-deps', 'tools', 'bregctl', '--format', 'json', 'webhook', 'replay', + '--runtime-config', '/config/breg/runtime.json', '--event-id', scenario['eventId'], + '--delivery-id', observed['deliveryId'], '--expected-generation', str(observed['generation'])], 'Explicit BREG dead-letter replay') + self.accepted(scenario) + require(delivery(scenario['eventId'])['generation'] > scenario.get('deadLetterGeneration', 0), 'Replay did not advance generation') + self.verify_effect(scenario) + scenario['phase'] = 'complete' + self.save() + + def after_accept(self): + scenario = self.scenario('after-accept') + if scenario['phase'] == 'complete': + return self.verify_effect(scenario) + if scenario['phase'] == 'prepared': + self.service('stop', 'destination') + try: + self.create(scenario) + self.accepted(scenario) + orders = self.poll(lambda: self.orders(scenario['recordId']), + lambda items: bool(items) and all(order['state'] in FAILURES for order in items), 'Post-acceptance workflow failure') + require(all(any(step['job'] == 'record-update' and step['exitReason'] != 'success' + for step in order['runs'][-1]['steps']) for order in orders), 'Failure did not occur at the synthetic destination') + scenario['phase'] = 'worker-failed' + self.save() + finally: + self.service('start', 'destination') + self.destination_ready() + if len(self.effects(scenario['recordId'])['effects']) == 1: + self.verify_effect(scenario) + scenario['phase'] = 'complete' + self.save() + return + require(len(self.effects(scenario['recordId'])['effects']) == 0, 'Destination changed during the forced failure') + retried = self.retry_failed(scenario['recordId']) + require(all(item['retriedJob'] == 'record-update' for item in retried), 'Retry unexpectedly reran evidence acquisition') + self.verify_effect(scenario) + scenario['phase'] = 'complete' + self.save() + + def pending_restart(self): + scenario = self.scenario('pending-restart') + if scenario['phase'] == 'complete': + return self.verify_effect(scenario) + if scenario.get('restartCompleted') and len(self.effects(scenario['recordId'])['effects']) == 1: + self.verify_effect(scenario) + scenario['phase'] = 'complete' + self.save() + return + self.service('stop', 'worker') + try: + self.create(scenario) + orders = self.accepted(scenario) + require(all(order['state'] not in SUCCESS | FAILURES for order in orders), 'Pending test had already executed') + require(len(self.effects(scenario['recordId'])['effects']) == 0, 'Pending work unexpectedly changed destination') + scenario['phase'] = 'accepted-pending' + self.save() + self.service('restart', 'lightning', 'bridge', 'breg', 'destination') + self.ready() + self.destination_ready() + self.registry_ready() + require(len(self.effects(scenario['recordId'])['effects']) == 0, 'Restart executed work without the worker') + scenario['restartCompleted'] = True + self.save() + finally: + self.service('start', 'worker') + self.verify_effect(scenario) + scenario['phase'] = 'complete' + self.save() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + actions = parser.add_mutually_exclusive_group(required=True) + actions.add_argument('--retry-smoke', action='store_true') + actions.add_argument('--inspect-smoke', action='store_true') + actions.add_argument('--replay-smoke', action='store_true') + actions.add_argument('--scenario', choices=['before-accept', 'after-accept', 'pending-restart', 'all']) + parser.add_argument('--timeout', type=int, default=180) + arguments = parser.parse_args() + require(10 <= arguments.timeout <= 600, 'Timeout must be between 10 and 600 seconds') + os.umask(0o077) + recovery = Recovery(arguments.timeout) + if arguments.inspect_smoke: + orders = recovery.orders(recovery.pilot.state['recordId']) + print(json.dumps({'status': 'inspected', 'workOrders': len(orders), 'states': [order['state'] for order in orders], + 'runs': sum(len(order['runs']) for order in orders), + 'latestRunStates': [order['runs'][-1]['state'] for order in orders if order['runs']], + 'latestQueues': [order['runs'][-1]['queue'] for order in orders if order['runs']]})) + elif arguments.retry_smoke: + results = recovery.retry_failed(recovery.pilot.state['recordId']) + print(json.dumps({'status': 'passed', 'retriedRuns': len(results)})) + elif arguments.replay_smoke: + print(json.dumps({'status': 'passed', 'replayedEvents': recovery.replay_smoke(), 'additionalDestinationEffects': 0})) + else: + scenarios = ['before-accept', 'after-accept', 'pending-restart'] if arguments.scenario == 'all' else [arguments.scenario] + for scenario in scenarios: + getattr(recovery, scenario.replace('-', '_'))() + print(json.dumps({'status': 'passed', 'checks': scenarios})) + + +if __name__ == '__main__': + try: + main() + except SmokeFailure as error: + print('Pilot recovery failed: ' + str(error), file=sys.stderr) + sys.exit(1) + except Exception: + print('Pilot recovery failed: unexpected response or configuration; details withheld', file=sys.stderr) + sys.exit(1) diff --git a/pilot/smoke.py b/pilot/smoke.py new file mode 100644 index 0000000..bccd59b --- /dev/null +++ b/pilot/smoke.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 +"""Exercise the retained synthetic pilot; all credentials and command output stay private. + +Run after provisioning: python3 pilot/smoke.py +Only after the main smoke passes, explicitly request retained-state restart proof: + python3 pilot/smoke.py --restart-recovery +No command deletes containers, volumes, records, or the persisted smoke identity. +""" +import argparse +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid + +REPO = Path(__file__).resolve().parent.parent +RUNTIME = REPO / 'pilot/agriculture/.runtime' +COMPOSE = [str(REPO / 'deployment/compose.sh')] +PROFILE = 'openfn-service' +SAFE_ID = re.compile(r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$') + + +class SmokeFailure(Exception): + """A deliberately value-free diagnostic suitable for the operator console.""" + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +def require(condition, message): + if not condition: + raise SmokeFailure(message) + + +def captured(args, label, timeout=120): + try: + result = subprocess.run(args, cwd=REPO, capture_output=True, text=True, timeout=timeout) + except (OSError, subprocess.TimeoutExpired): + raise SmokeFailure(label + ' did not complete') from None + if result.returncode: + raise SmokeFailure(label + ' failed; subprocess output was withheld') + return result.stdout + + +def json_output(value, label): + try: + return json.loads(value) + except (ValueError, TypeError): + raise SmokeFailure(label + ' returned an invalid response') from None + + +def private_write(path, value): + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + temporary = path.with_suffix('.tmp') + fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, 'w') as handle: + json.dump(value, handle, indent=2) + handle.write('\n') + temporary.chmod(0o600) + os.replace(temporary, path) + path.chmod(0o600) + + +def read_secret(path): + try: + value = path.read_text().strip() + except OSError: + raise SmokeFailure('Required private pilot configuration is not ready') from None + require(bool(value), 'Required private pilot configuration is empty') + return value + + +def http_json(url, *, method='GET', body=None, headers=None, timeout=20, parse_json=True): + request = urllib.request.Request(url, method=method, + data=None if body is None else json.dumps(body).encode(), + headers={'Accept': 'application/json', **({'Content-Type': 'application/json'} if body is not None else {}), **(headers or {})}) + try: + with urllib.request.build_opener(NoRedirect()).open(request, timeout=timeout) as response: + return response.status, json_output(response.read().decode(), 'HTTP service') if parse_json else None + except urllib.error.HTTPError as error: + # Bodies may contain subjects, workflow inputs, or diagnostic details. + # Keep only the status for failures; never render the response text. + return error.code, None + except (OSError, urllib.error.URLError, TimeoutError): + raise SmokeFailure('A pilot HTTP exchange did not complete') from None + + +def operation_path(operation, record_id=None): + path = operation.get('path') + require(isinstance(path, str) and path.startswith('/v1/') and not any(value in path for value in ['?', '#', '..', '\\']), 'Metadata did not provide an expected relative Registry route') + if record_id is not None: + require(bool(SAFE_ID.fullmatch(record_id)), 'Registry returned an invalid record identifier') + path = path.replace('{record_id}', record_id) + require('{' not in path and '}' not in path, 'Metadata requires an unsupported route parameter') + return path + + +class Pilot: + def __init__(self, timeout=180): + self.timeout = timeout + self.secrets = RUNTIME / 'openfn/secrets' + self.project = json_output(read_secret(self.secrets / 'project.json'), 'Pilot project') + self.webhook_key = read_secret(self.secrets / 'webhook-api-key') + self.admin_token = read_secret(self.secrets / 'api-token') + self.state_path = RUNTIME / 'smoke-state.json' + if self.state_path.exists(): + self.state = json_output(read_secret(self.state_path), 'Retained smoke state') + require(self.state.get('schema') == 'registry-openfn-smoke/v1', 'Retained smoke state has an unsupported schema') + else: + # One fresh identity, persisted before effects. Every operation key and + # synthetic selector thereafter derives deterministically from it. + run_id = uuid.uuid4().hex[:16] + self.state = {'schema': 'registry-openfn-smoke/v1', 'runId': run_id, + 'registration': {'submissionId': 'smoke-registration-' + run_id, + 'localIdentifier': 'SYNTHETIC-SMOKE-' + run_id, + 'name': 'Synthetci Smoke Holding ' + run_id}, + 'correctedName': 'Synthetic Smoke Holding ' + run_id, + 'supportingReference': 'SYNTHETIC-SMOKE-SUPPORT-' + run_id, + 'reason': 'Synthetic acceptance spelling correction', 'phase': 'prepared'} + self.save() + + def save(self): + private_write(self.state_path, self.state) + + def token(self): + value = captured(COMPOSE + ['run', '--rm', '--no-deps', 'tools', 'mint', 'token', + '--url', 'http://127.0.0.1:8091/token', '--client-id', PROFILE, + '--key', '/config/breg/clients/openfn-service/signing-p256-private-jwk'], 'Mint authentication').strip() + require(value.count('.') == 2 and not any(character.isspace() for character in value), 'Mint did not return one access token') + return value + + def breg(self, path, method='GET', body=None, headers=None): + separator = '&' if '?' in path else '?' + url = 'http://127.0.0.1:4002' + path + separator + urllib.parse.urlencode({'accessProfile': PROFILE}) + status, value = http_json(url, method=method, body=body, + headers={'Authorization': 'Bearer ' + self.token(), **(headers or {})}) + require(200 <= status < 300, 'Registry acceptance request was refused (HTTP ' + str(status) + ')') + return value + + def metadata(self): + return self.breg('/v1/registry') + + def select(self, metadata, entity, kind): + matches = [value for value in metadata.get('operations', []) if value.get('sourceEntity') == entity + and value.get('operation') == kind and value.get('accessProfile') == PROFILE] + require(len(matches) == 1, 'Caller-filtered metadata has no unique required operation') + return matches[0] + + def farm(self): + metadata = self.metadata() + lookup = self.select(metadata, 'farm', 'lookup') + selectors = [value for value in lookup.get('selectors', []) if value.get('id') == 'by-local-identifier'] + require(len(selectors) == 1 and selectors[0].get('requestFields') == ['localIdentifier'], 'Farm lookup selector differs from the pilot contract') + return self.breg(operation_path(lookup), 'POST', {'selector': 'by-local-identifier', 'values': {'localIdentifier': self.state['registration']['localIdentifier']}}) + + def current_request(self): + metadata = self.metadata() + get = self.select(metadata, 'name-correction', 'get') + return self.breg(operation_path(get, self.state['requestId'])) + + def poll_work_order(self, work_order_id, expect_success): + require(isinstance(work_order_id, str) and bool(SAFE_ID.fullmatch(work_order_id)), 'OpenFn returned an invalid work-order identifier') + end = time.monotonic() + self.timeout + while time.monotonic() < end: + status, value = http_json('http://127.0.0.1:4000/api/work_orders/' + work_order_id, + headers={'Authorization': 'Bearer ' + self.admin_token}) + require(status == 200, 'OpenFn work-order inspection failed') + state = value.get('data', {}).get('attributes', {}).get('state') + if state in ['success', 'completed']: + require(expect_success, 'Conflicting input unexpectedly succeeded') + return + if state in ['failed', 'crashed', 'killed', 'cancelled', 'exception', 'lost']: + require(not expect_success, 'OpenFn workflow did not succeed') + return + time.sleep(1) + raise SmokeFailure('OpenFn work order did not reach a terminal state within the deadline') + + def webhook(self, name, data, expect_success=True): + trigger = self.project.get('triggerIds', {}).get(name) + require(isinstance(trigger, str) and bool(SAFE_ID.fullmatch(trigger)), 'Private project configuration lacks a trigger') + status, value = http_json('http://127.0.0.1:4000/i/' + trigger, method='POST', body=data, + headers={'x-api-key': self.webhook_key}, timeout=self.timeout) + if not expect_success and status == 422: + return None + require(200 <= status < 300, 'OpenFn webhook returned an unexpected HTTP status') + # after_completion returns the final state. before_start returns a + # work_order_id, whose documented JSONAPI state is polled explicitly. + if isinstance(value, dict) and 'work_order_id' in value: + self.poll_work_order(value['work_order_id'], expect_success) + else: + require(expect_success, 'Conflicting input unexpectedly succeeded') + return value + + def destination(self): + script = "const fs=require('node:fs');fetch('http://127.0.0.1:8082/status',{headers:{'x-destination-key':fs.readFileSync('/config/destination/api-key','utf8')}}).then(async r=>{if(!r.ok)process.exit(1);process.stdout.write(JSON.stringify(await r.json()))}).catch(()=>process.exit(1))" + return json_output(captured(COMPOSE + ['exec', '-T', 'destination', 'node', '-e', script], 'Destination status'), 'Destination status') + + def wait_destination(self, expected_effects): + end = time.monotonic() + self.timeout + while time.monotonic() < end: + rows = self.destination_rows() + effects = rows.get('effects') + require(isinstance(effects, list), 'Destination storage diagnostics have an unexpected shape') + count = len(effects) + if count == expected_effects: + require(len(rows.get('records', [])) == 1, 'Destination has no unique current record for the smoke') + return rows + require(count < expected_effects, 'Destination received an unexpected additional event for this smoke record') + time.sleep(2) + raise SmokeFailure('Verified destination effect did not arrive within the deadline') + + def destination_rows(self): + # Private captured inspection proves the stored DTO contains only the + # verified boolean, references and revision, with one effect per event. + script = "const {DatabaseSync}=require('node:sqlite');const d=new DatabaseSync('/data/destination.sqlite',{readOnly:true});const id=process.argv[1];process.stdout.write(JSON.stringify({records:d.prepare('SELECT * FROM records WHERE record_id=?').all(id),effects:d.prepare('SELECT * FROM effects WHERE record_id=?').all(id)}));d.close()" + return json_output(captured(COMPOSE + ['exec', '-T', 'destination', 'node', '--no-warnings', '-e', script, self.state['recordId']], 'Private destination storage inspection'), 'Destination storage') + + def correction(self): + return {'submissionId': 'smoke-correction-' + self.state['runId'], 'recordId': self.state['recordId'], + 'name': self.state['correctedName'], 'reason': self.state['reason'], 'supportingReference': self.state['supportingReference']} + + def recover_request_id(self, response): + # Synchronous form workflows return minimal state; accept its documented + # data wrapper, including the webhook's wrapper around the final state. + for value in [response.get('data') if isinstance(response, dict) else None, + response.get('data', {}).get('data') if isinstance(response, dict) and isinstance(response.get('data'), dict) else None]: + if isinstance(value, dict) and isinstance(value.get('requestId'), str) and SAFE_ID.fullmatch(value['requestId']): + return value['requestId'] + # No list grant is added for diagnostics. An exact replay under the same + # principal, profile, payload and durable key returns the original draft. + metadata = self.metadata() + operation = self.select(metadata, 'name-correction', 'create') + correction = self.correction() + fields = {'record': correction['recordId'], **{field: correction[field] for field in ['name', 'reason', 'supportingReference']}} + value = self.breg(operation_path(operation), 'POST', {'data': fields}, + {'Idempotency-Key': 'correction:' + correction['submissionId']}) + identifier = value.get('data', {}).get('recordIdentifier') + require(isinstance(identifier, str) and bool(SAFE_ID.fullmatch(identifier)), 'Correction replay did not return its original request identifier') + return identifier + + def inspect_content(self, record): + current = record.get('data', {}).get('domainData', {}) + expected = {'record': self.state['recordId'], 'name': self.state['correctedName'], + 'reason': self.state['reason'], 'supportingReference': self.state['supportingReference']} + require(all(current.get(key) == value for key, value in expected.items()), 'Current correction differs from the retained submitted event') + + def review(self, action): + captured(COMPOSE + ['run', '--rm', '--no-deps', 'tools', 'python3', '/opt/pilot/agriculture/review.py', action, self.state['requestId']], 'Explicit reviewer ' + action) + + def verify_retained(self): + farm = self.farm() + require(farm['data']['recordIdentifier'] == self.state['recordId'], 'Registration replay changed record identity') + require(farm['data']['domainData'].get('name') == self.state['correctedName'], 'Applied correction is not retained') + current = self.current_request() + self.inspect_content(current) + require(current['data'].get('request', {}).get('bregState') == 'applied', 'Reviewed correction is not applied') + self.wait_destination(2) + rows = self.destination_rows() + require(len(rows.get('records', [])) == 1 and len(rows.get('effects', [])) == 2, 'Destination did not retain exactly one effect per farm event') + require(all(row.get('outcome') == 'applied' for row in rows['effects']), 'Destination accepted an unexpected stale event') + require(rows['records'][0].get('registered') == 1, 'Destination did not retain the verified boolean') + require(len({row['event_id'] for row in rows['effects']}) == 2, 'Destination contains a repeated event identity') + serialized = json.dumps(rows) + require(self.state['registration']['name'] not in serialized and self.state['correctedName'] not in serialized, + 'Destination contains a raw farm name') + + def run(self): + if 'baseline' not in self.state: + self.state['baseline'] = self.destination() + self.save() + self.webhook('registration', self.state['registration']) + farm = self.farm() + record_id = farm.get('data', {}).get('recordIdentifier') + require(isinstance(record_id, str) and bool(SAFE_ID.fullmatch(record_id)), 'Registration lookup returned no record identifier') + if 'recordId' in self.state: + require(self.state['recordId'] == record_id, 'Retained registration identity changed') + self.state['recordId'] = record_id + self.save() + self.webhook('registration', self.state['registration']) + self.webhook('registration', {**self.state['registration'], 'name': 'Conflicting synthetic retry'}, expect_success=False) + if self.state.get('phase') == 'complete': + self.webhook('correction', self.correction()) + self.verify_retained() + return + require(farm['data']['domainData'].get('name') in [self.state['registration']['name'], self.state['correctedName']], 'Farm state differs from the retained smoke input') + if farm['data']['domainData'].get('name') == self.state['registration']['name']: + self.wait_destination(1) + response = self.webhook('correction', self.correction()) + request_id = self.recover_request_id(response) + if 'requestId' in self.state: + require(self.state['requestId'] == request_id, 'Correction replay changed request identity') + self.state['requestId'] = request_id + self.save() + self.webhook('correction', self.correction()) + current = self.current_request() + self.inspect_content(current) + state = current['data'].get('request', {}).get('bregState') + require(state in ['submitted', 'approved', 'applied'], 'Correction is not in an expected review state') + if state == 'submitted': + require(self.farm()['data']['domainData'].get('name') == self.state['registration']['name'], 'Submission changed the governed farm before approval') + self.review('inspect') + self.review('approve') + require(self.farm()['data']['domainData'].get('name') == self.state['registration']['name'], 'Approval unexpectedly applied the correction') + self.state['phase'] = 'approved' + self.save() + state = 'approved' + if state == 'approved': + require(self.farm()['data']['domainData'].get('name') == self.state['registration']['name'], 'Farm changed before explicit application') + self.review('inspect') + self.review('apply') + self.verify_retained() + self.state['phase'] = 'complete' + self.save() + self.webhook('registration', self.state['registration']) + self.webhook('correction', self.correction()) + self.verify_retained() + + def restart_recovery(self): + require(self.state.get('phase') == 'complete', 'Complete the main acceptance smoke before restart recovery') + # Only restart these existing pilot services. Keep databases, Mint's + # shared network namespace, and every named volume intact. + captured(COMPOSE + ['restart', 'breg', 'worker', 'bridge', 'destination', 'lightning'], 'Retained-state pilot restart', timeout=180) + end = time.monotonic() + self.timeout + while time.monotonic() < end: + try: + status, _ = http_json('http://127.0.0.1:4000/health_check', timeout=5, parse_json=False) + if status == 200: + break + except SmokeFailure: + pass + time.sleep(2) + else: + raise SmokeFailure('Restarted OpenFn did not become ready') + self.run() + self.state['restartRecovery'] = 'passed' + self.save() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--restart-recovery', action='store_true') + parser.add_argument('--timeout', type=int, default=180) + args = parser.parse_args() + os.umask(0o077) + require(10 <= args.timeout <= 600, 'Timeout must be between 10 and 600 seconds') + pilot = Pilot(args.timeout) + if args.restart_recovery: + pilot.restart_recovery() + else: + pilot.run() + print(json.dumps({'status': 'passed', 'checks': ['registration', 'exact-replay', 'conflicting-retry', + 'governed-submit', 'separate-approval-and-application', 'verified-event-effects', 'retained-replay'], + 'restartRecovery': pilot.state.get('restartRecovery', 'not-run')})) + + +if __name__ == '__main__': + try: + main() + except SmokeFailure as error: + print('Pilot smoke failed: ' + str(error), file=sys.stderr) + sys.exit(1) + except Exception: + print('Pilot smoke failed: unexpected response or local configuration; details withheld', file=sys.stderr) + sys.exit(1) diff --git a/pilot/test/evidence-workflow.test.js b/pilot/test/evidence-workflow.test.js new file mode 100644 index 0000000..69ba5cc --- /dev/null +++ b/pilot/test/evidence-workflow.test.js @@ -0,0 +1,216 @@ +import assert from 'node:assert/strict'; +import { createHash, generateKeyPairSync, randomBytes, sign } from 'node:crypto'; +import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'; +import http from 'node:http'; +import { tmpdir } from 'node:os'; +import { resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; +import compile from '@openfn/compiler'; +import run from '@openfn/runtime'; +import { projectDocument } from '../lightning/project.mjs'; + +const root = fileURLToPath(new URL('../../', import.meta.url)); +const bindings = { + accessProfile: 'pilot', createFarmOperation: 'create', createCorrectionOperation: 'correct', + requirement: 'farm-registered', registeredConcept: 'urn:example:concept:registered', +}; +const revision = `sha256:${'a'.repeat(64)}`; +const subjectBinding = `urn:evidence:subject:v1_${randomBytes(32).toString('base64url')}`; +const traceId = '4bf92f3577b34da6a3ce929d0e0e4736'; +const spec = { + requirement: 'urn:example:requirement:farm-registered', purpose: 'registration-check', + audience: 'urn:example:openfn', evidenceType: 'urn:example:evidence-type:registered', + issuedBy: 'urn:example:agricultural-registry', providedBy: 'urn:example:evidence', +}; +const envelope = () => ({ + event: { specversion: '1.0', source: 'urn:registrystack:registry:agricultural-holdings:instance:pilot', + id: '22222222-2222-4222-8222-222222222222', type: 'registry.record.committed', + time: '2026-01-01T00:00:00Z', dataschema: 'urn:example:committed-event' }, + delivery: { generation: 3, attempt: 2, time: '2026-09-09T00:00:00Z', idempotencyKey: `sha256:${'c'.repeat(64)}` }, + data: { entity: 'farm', recordId: '11111111-1111-4111-8111-111111111111', revision: 7, + trigger: 'patched', packageRevision: `sha256:${'b'.repeat(64)}`, + values: { 'local-identifier': 'synthetic-farm-selector' } }, +}); + +function keyPair() { + const { publicKey, privateKey } = generateKeyPairSync('ec', { namedCurve: 'prime256v1' }); + const jwk = publicKey.export({ format: 'jwk' }); + const kid = createHash('sha256').update(JSON.stringify({ crv: jwk.crv, kty: jwk.kty, x: jwk.x, y: jwk.y })).digest('base64url'); + return { privateKey, jwks: { keys: [{ ...jwk, kid, alg: 'ES256' }] }, + privateJwk: { ...privateKey.export({ format: 'jwk' }), kid, alg: 'ES256' }, kid }; +} + +async function fixture(outcome = 'true') { + const key = keyPair(); + const clientKey = keyPair(); + const directory = mkdtempSync(resolve(tmpdir(), 'openfn-pilot-evidence-')); + const requests = []; + let origin; + const json = (res, value, status = 200, mediaType = 'application/json') => { + res.writeHead(status, { 'content-type': mediaType, traceparent: `00-${traceId}-00f067aa0ba902b7-01` }); + res.end(JSON.stringify(value)); + }; + const server = http.createServer((req, res) => { + const chunks = []; + req.on('data', chunk => chunks.push(chunk)); + req.on('end', () => { + const body = Buffer.concat(chunks).toString('utf8'); + requests.push({ path: req.url, body, authorization: req.headers.authorization }); + if (req.url === '/.well-known/oauth-protected-resource') return json(res, { + resource: origin, authorization_servers: [origin], jwks_uri: `${origin}/.well-known/evidence/jwks.json`, bearer_methods_supported: ['header'], + }); + if (req.url === '/.well-known/oauth-authorization-server') return json(res, { + issuer: origin, token_endpoint: `${origin}/token`, grant_types_supported: ['client_credentials'], token_endpoint_auth_methods_supported: ['private_key_jwt'], + }); + if (req.url === '/.well-known/evidence/jwks.json') return json(res, key.jwks, 200, 'application/jwk-set+json'); + if (req.url === '/token') return json(res, { access_token: 'synthetic-evidence-token', token_type: 'Bearer', expires_in: 300 }); + if (req.url === '/v1/evidence-definitions') return json(res, { + schema: 'registry.evidence-definitions/v1', assuranceProfile: 'local', audience: spec.audience, issuedBy: spec.issuedBy, providedBy: spec.providedBy, + definitions: [{ handle: bindings.requirement, requirement: spec.requirement, configurationRevision: revision, + kind: 'criterion', evidenceType: spec.evidenceType, purpose: spec.purpose, responseFormats: ['signed-jws'], + referenceFrameworks: ['urn:example:framework:registration'], + subjects: [{ role: 'farm', cardinality: 'one', selector: { profile: 'farm-id', valueOrigin: 'request', + fields: [{ type: 'string', name: 'local-identifier', minimumBytes: 1, maximumBytes: 200 }] } }], + concepts: [{ handle: 'registered', concept: bindings.registeredConcept, required: true, form: 'boolean' }] }], + }); + if (req.url === '/updates') return json(res, { status: 'applied', revision: JSON.parse(body).revision }); + if (req.url === '/v1/evidence') { + if (outcome === 'unavailable') return json(res, { + type: 'https://id.registrystack.org/problems/registry-evidence/evidence/unavailable', + title: 'Evidence could not be produced', status: 422, + detail: 'evidence could not be produced for this request', code: 'evidence.unavailable', traceId, + }, 422, 'application/problem+json'); + const now = new Date().toISOString().replace(/\.\d+Z$/, 'Z'); + const assertion = { schema: 'registry.assertion-evidence/v1', assuranceProfile: 'local', subjectBinding: 'audience-scoped', + requestNonce: JSON.parse(body).requestNonce, id: 'urn:example:assertion:current-observation', type: 'Evidence', + supportsRequirement: spec.requirement, isConformantTo: spec.evidenceType, issuedBy: spec.issuedBy, providedBy: spec.providedBy, + issuedAt: now, observedAt: now, validUntil: new Date(Date.now() + 60_000).toISOString().replace(/\.\d+Z$/, 'Z'), + purpose: spec.purpose, audience: spec.audience, configurationRevision: revision, + subjects: [{ role: 'farm', binding: subjectBinding }], + supportedValues: outcome === 'missing-value' ? [] : [{ providesValueFor: bindings.registeredConcept, value: outcome !== 'false' }], + }; + const protectedPart = Buffer.from(JSON.stringify({ alg: 'ES256', kid: key.kid, typ: 'evidence+jws', cty: 'application/evidence+json' })).toString('base64url'); + const payload = Buffer.from(JSON.stringify(assertion)).toString('base64url'); + const signature = sign('sha256', Buffer.from(`${protectedPart}.${payload}`), { key: key.privateKey, dsaEncoding: 'ieee-p1363' }).toString('base64url'); + return json(res, { protected: protectedPart, payload, signature }, 200, 'application/jose+json'); + } + json(res, {}, 404); + }); + }); + await new Promise((done, reject) => { server.once('error', reject); server.listen(0, '127.0.0.1', done); }); + origin = `http://127.0.0.1:${server.address().port}`; + const profilePath = resolve(directory, 'client.json'); + writeFileSync(profilePath, JSON.stringify({ schema: 'registry.evidence-client-profile/v1', baseUrl: origin, + clientId: 'openfn-pilot', privateKey: { source: 'file', path: 'unused.jwk' }, + trust: { type: 'local-loopback-discovery' }, contracts: { type: 'published' } }), { mode: 0o600 }); + return { requests, destinationConfiguration: { baseUrl: origin, apiKey: 'synthetic-destination-key' }, configuration: { evidence: { profilePath, privateKeyJwk: clientKey.privateJwk } }, + close: async () => { await new Promise(done => server.close(done)); rmSync(directory, { recursive: true, force: true }); } }; +} + +async function executeEvidence(configuration, data, destinationConfiguration) { + const workflow = projectDocument(bindings).workflows.find(workflow => workflow.name === 'Agriculture committed'); + const job = workflow.jobs.find(job => job.name === 'verify-registration'); + const edge = workflow.edges.find(edge => edge.source_job_id === job.id); + assert.equal(edge.condition_type, 'on_job_success'); + // Lightning injects operation imports from the selected adaptor. Keep the + // exact stored job body here so missing common reexports fail as in a worker. + const { code } = compile(job.body, { + 'add-imports': { adaptors: [{ name: '@openfn/language-registry-evidence', exportAll: true }] }, + }); + assert.match(code, /import \{[^}]*fn[^}]*\} from ["']@openfn\/language-registry-evidence["']/); + assert.doesNotMatch(code, /from ["']@openfn\/language-common["']/); + const destination = destinationConfiguration + ? compile(workflow.jobs.find(job => job.name === 'record-update').body, { + 'add-imports': { adaptors: [{ name: '@openfn/language-http', exportAll: true }] }, + }).code + : compile(`import { fn } from '@openfn/language-common'; fn(state => ({...state, destinationReached: true}));`).code; + return run({ workflow: { steps: [ + { id: 'verify', expression: code, next: { destination: edge.condition_type } }, + { id: 'destination', expression: destination, configuration: destinationConfiguration }, + ], start: 'verify' }, options: { start: 'verify' } }, { configuration, data }, { + linker: { modules: { '@openfn/language-common': { path: resolve(root, 'node_modules/@openfn/language-common') }, + '@openfn/language-registry-evidence': { path: resolve(root, 'packages/registry-evidence') }, + '@openfn/language-http': { path: resolve(root, 'node_modules/@openfn/language-http') } }, + cacheKey: `pilot-evidence-${Date.now()}-${Math.random()}` }, + }); +} + +test('committed workflow emits only event identity/revision and verified current registration', async () => { + const service = await fixture(); + try { + const input = envelope(); + const result = await executeEvidence(service.configuration, input); + assert.equal(result.errors, undefined); + assert.equal(result.destinationReached, true); + assert.deepEqual(Object.keys(result.integration).sort(), ['committedRevision', 'deliveryGeneration', 'eventId', 'eventSource', 'evidenceObservedAt', 'evidenceTraceId']); + assert.equal(result.integration.committedRevision, 7); + assert.equal(result.integration.deliveryGeneration, 3); + assert.equal(result.integration.eventSource, input.event.source); + assert.equal(result.integration.eventId, input.event.id); + assert.equal(result.integration.evidenceTraceId, traceId); + assert.ok(Date.parse(result.integration.evidenceObservedAt) > Date.parse(input.event.time)); + for (const forbidden of ['jws', 'assertion', 'privateKeyJwk', 'subject_continuity', 'requestNonce']) { + assert.equal(JSON.stringify(result).includes(forbidden), false); + } + assert.deepEqual(result.data, { source: input.event.source, eventId: input.event.id, effect: 'record-sync', + recordId: input.data.recordId, revision: 7, values: { registered: true } }); + assert.equal(JSON.stringify(result).includes('synthetic-farm-selector'), false); + assert.equal(JSON.stringify(result).includes('synthetic-evidence-token'), false); + assert.equal(JSON.stringify(result).includes(revision), false); + assert.equal('configuration' in result, false); + const requests = service.requests.filter(request => request.path === '/v1/evidence'); + assert.equal(requests.length, 1); + assert.deepEqual(JSON.parse(requests[0].body).subjects, [{ role: 'farm', selector: { profile: 'farm-id', values: { 'local-identifier': 'synthetic-farm-selector' } } }]); + } finally { await service.close(); } +}); + +for (const outcome of ['false', 'unavailable', 'missing-value']) { + test(`${outcome} evidence never reaches destination success`, async () => { + const service = await fixture(outcome); + try { + const result = await executeEvidence(service.configuration, envelope()); + assert.ok(result.errors?.verify); + assert.equal(result.destinationReached, undefined); + assert.equal(result.data.effect, undefined); + assert.equal('configuration' in result, false); + assert.equal(service.requests.filter(request => request.path === '/v1/evidence').length, 1); + } finally { await service.close(); } + }); +} + +test('missing event metadata, selector, or revision refuses before Evidence I/O', async () => { + const service = await fixture(); + try { + for (const mutate of [input => { delete input.event.id; }, input => { delete input.data.values['local-identifier']; }, input => { input.data.revision = undefined; }, input => { input.delivery.generation = undefined; }]) { + const input = envelope(); mutate(input); + const result = await executeEvidence(service.configuration, input); + assert.ok(result.errors?.verify); + assert.equal(result.destinationReached, undefined); + } + assert.equal(service.requests.length, 0); + } finally { await service.close(); } +}); + + +test('destination job preserves integration metadata without adding it to the HTTP payload', async () => { + const service = await fixture(); + try { + const input = envelope(); + const result = await executeEvidence(service.configuration, input, service.destinationConfiguration); + assert.equal(result.errors, undefined); + assert.deepEqual(result.data, { status: 'applied', revision: 7 }); + assert.equal(result.integration.committedRevision, 7); + assert.equal(result.integration.deliveryGeneration, 3); + assert.equal(result.integration.eventId, input.event.id); + assert.equal(result.integration.eventSource, input.event.source); + assert.equal(result.integration.evidenceTraceId, traceId); + assert.ok(Date.parse(result.integration.evidenceObservedAt) > Date.parse(input.event.time)); + const updates = service.requests.filter(request => request.path === '/updates'); + assert.equal(updates.length, 1); + assert.deepEqual(JSON.parse(updates[0].body), { source: input.event.source, eventId: input.event.id, effect: 'record-sync', recordId: input.data.recordId, revision: 7, values: { registered: true } }); + for (const forbidden of ['jws', 'assertion', 'privateKeyJwk', 'synthetic-destination-key', 'synthetic-farm-selector']) { + assert.equal(JSON.stringify(result).includes(forbidden), false); + } + } finally { await service.close(); } +}); diff --git a/pilot/test/project.test.js b/pilot/test/project.test.js new file mode 100644 index 0000000..dc6e171 --- /dev/null +++ b/pilot/test/project.test.js @@ -0,0 +1,271 @@ +import assert from 'node:assert/strict'; +import http from 'node:http'; +import { isDeepStrictEqual } from 'node:util'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; +import compile from '@openfn/compiler'; +import run from '@openfn/runtime'; +import * as bregAdaptor from '../../packages/registry-breg/src/index.js'; +import { projectDocument, projectId, triggerIds, id } from '../lightning/project.mjs'; +import { metadata as baseMetadata, record as baseRecord, ETAG, ACTION_ETAG, ID, TRACE } from '../../packages/registry-breg/test/fixtures.js'; + +const PROFILE = 'editor'; +const REQUEST_ID = '00000000-0000-4000-8000-000000000002'; +const bindings = { accessProfile: PROFILE, createFarmOperation: 'records.farm.create', createCorrectionOperation: 'records.name-correction.create', requirement: 'urn:synthetic:registered', registeredConcept: 'urn:synthetic:registered-concept' }; +const correctionInput = { submissionId: 'correction-001', recordId: ID, name: 'Synthetic Farm Corrected', reason: 'Synthetic correction', supportingReference: 'SYNTHETIC-001' }; +const correctionData = input => ({ record: input.recordId, name: input.name, reason: input.reason, supportingReference: input.supportingReference }); +const packagePath = fileURLToPath(new URL('../../packages/registry-breg', import.meta.url)); +let cacheId = 0; + +function compileJob(body) { + return compile(body, { 'add-imports': { adaptors: [{ name: '@openfn/language-registry-breg', exports: Object.keys(bregAdaptor) }] } }).code; +} +async function runJob(name, baseUrl, data) { + const workflow = projectDocument(bindings).workflows.find(item => item.name === `Agriculture ${name}`); + const code = compileJob(workflow.jobs[0].body); + return run({ workflow: { steps: [{ id: 'pilot', expression: code }], start: 'pilot' }, options: { start: 'pilot' } }, + { configuration: { breg: { baseUrl, authorization: { static: 'synthetic-pilot-token' } } }, data }, + { linker: { modules: { '@openfn/language-registry-breg': { path: packagePath }, '@openfn/language-common': { path: fileURLToPath(new URL('../../node_modules/@openfn/language-common', import.meta.url)) } }, cacheKey: `pilot-project-${process.pid}-${cacheId++}` } }); +} + +function metadata(entity, route, fields) { + const value = baseMetadata(); + const fieldId = name => name.replace(/[A-Z]/g, letter => "-" + letter.toLowerCase()); + const fieldIds = Object.keys(fields).map(fieldId); + value.entities[0].id = entity; + value.entities[0].route = route; + value.entities[0].readableFields = fieldIds; + value.entities[0].schema = `/v1/schemas/${entity}`; + value.entities[0].operations.forEach(operation => { operation.accessProfile = PROFILE; }); + value.operations.forEach(operation => { + operation.id = operation.id.replace('company', entity); + operation.path = operation.path.replace('companies', route); + operation.sourceEntity = entity; + operation.responseEntity = entity; + operation.accessProfile = PROFILE; + operation.titleFields = [fieldIds[0]]; + operation.fields = Object.entries(fields).map(([field, type]) => ({ id: fieldId(field), apiName: field, label: field, schema: { type }, required: true, nullable: false, readOnly: false, removable: false })); + operation.readableFields = fieldIds; + operation.createWritableFields = operation.operation === 'create' ? fieldIds : []; + operation.patchWritableFields = operation.operation === 'patch' ? fieldIds : []; + }); + return value; +} +function record(entity, recordId, data, lifecycleState) { + const value = baseRecord(Boolean(lifecycleState)); + value.data.recordIdentifier = recordId; + value.data.snapshot = `breg1_${recordId}`; + value.data.domainData = data; + value.meta.entityTypeIdentifier = entity; + if (lifecycleState) { + value.data.request.bregState = lifecycleState; + value.data.request.actions = lifecycleState === 'draft' ? [{ operation: 'submit_request', method: 'POST', href: `/v1/records/name-corrections/${recordId}/actions/submit?accessProfile=${PROFILE}`, ifMatch: ACTION_ETAG }] : []; + } + return value; +} +async function fixture({ entity = 'farm', currentState = 'draft', currentData, storedData, failStatus } = {}) { + const requests = []; + const route = entity === 'farm' ? 'farms' : 'name-corrections'; + const recordId = entity === 'farm' ? ID : REQUEST_ID; + const fields = entity === 'farm' ? { localIdentifier: 'string', name: 'string' } : { record: 'string', name: 'string', reason: 'string', supportingReference: 'string' }; + const metadataValue = metadata(entity, route, fields); + let stored = storedData; + let mutationCount = 0; + const server = http.createServer(async (req, res) => { + let body = ''; + for await (const chunk of req) body += chunk; + requests.push({ method: req.method, url: req.url, headers: req.headers, body }); + res.setHeader('traceparent', `00-${TRACE}-00f067aa0ba902b7-01`); + res.setHeader('content-type', 'application/json'); + res.setHeader('cache-control', 'no-store'); + res.setHeader('vary', 'authorization, accept'); + if (failStatus) { + res.statusCode = failStatus; + res.setHeader('content-type', 'application/problem+json'); + res.end(JSON.stringify({ type: 'about:blank', status: failStatus, code: 'auth.forbidden', traceId: TRACE, title: 'secret-failure-canary', detail: 'secret-failure-canary' })); + return; + } + if (req.url.includes('/v1/registry')) return res.end(JSON.stringify(metadataValue)); + if (req.url.includes('/actions/')) { + mutationCount++; + currentState = 'submitted'; + const value = record(entity, recordId, stored, 'draft'); + return res.end(JSON.stringify({ id: recordId, revision: 8, snapshot: `breg1_${recordId}`, request: { bregState: 'submitted', proposalVersion: 7, effectDigest: value.data.request.effectDigest, application: null } })); + } + res.setHeader('link', `; rel="profile", ; rel="describedby"`); + res.setHeader('etag', ETAG); + if (req.method === 'POST') { + const incoming = JSON.parse(body).data; + if (!stored) { stored = incoming; mutationCount++; } + else if (!isDeepStrictEqual(incoming, stored)) { + res.statusCode = 409; + res.setHeader('content-type', 'application/problem+json'); + return res.end(JSON.stringify({ type: 'about:blank', status: 409, code: 'mutation.conflict', traceId: TRACE })); + } + res.statusCode = 201; + res.setHeader('location', `/tenant/v1/records/${route}/${recordId}`); + return res.end(JSON.stringify(record(entity, recordId, stored))); + } + return res.end(JSON.stringify(record(entity, recordId, currentData ?? stored, currentState))); + }); + await new Promise((resolve, reject) => { server.once('error', reject); server.listen(0, '127.0.0.1', resolve); }); + return { requests, mutationCount: () => mutationCount, baseUrl: `http://127.0.0.1:${server.address().port}/tenant`, close: () => new Promise(resolve => server.close(resolve)) }; +} + +function assertClean(result) { + assert.equal('configuration' in result, false); + assert.ok(!JSON.stringify(result).includes('synthetic-pilot-token')); + assert.ok(!JSON.stringify(result).includes('secret-failure-canary')); +} + +test('project graph has deterministic unique IDs and starts disabled with webhook authentication required', () => { + const first = projectDocument(bindings); + assert.deepEqual(first, projectDocument(bindings)); + assert.equal(first.id, projectId); + const identifiers = [first.id]; + for (const workflow of first.workflows) { + identifiers.push(workflow.id, ...workflow.jobs.map(job => job.id), ...workflow.triggers.map(trigger => trigger.id), ...workflow.edges.map(edge => edge.id)); + assert.equal(workflow.triggers[0].enabled, false); + assert.equal(workflow.triggers[0].has_auth_method, true); + assert.ok(workflow.jobs.every(job => job.project_credential_id === null)); + const jobs = new Set(workflow.jobs.map(job => job.id)); + for (const edge of workflow.edges) assert.ok(jobs.has(edge.target_job_id)); + } + assert.equal(identifiers.length, new Set(identifiers).size); + assert.ok(identifiers.every(value => /^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(value))); + assert.equal(triggerIds.registration, id('registration:trigger')); + const configured = projectDocument(bindings, { breg: id('credential:breg'), evidence: id('credential:evidence'), destination: id('credential:destination') }, true); + assert.ok(configured.workflows.every(workflow => workflow.triggers[0].enabled)); + assert.equal(configured.workflows[0].jobs[0].project_credential_id, id('credential:breg')); +}); + +test('binding strings are JavaScript literals and cannot inject operations', () => { + const hostile = "synthetic'); throw new Error('injected'); //\n\u2028\"\\"; + const project = projectDocument(Object.fromEntries(Object.keys(bindings).map(key => [key, hostile]))); + for (const workflow of project.workflows) { + for (const job of workflow.jobs) assert.doesNotThrow(() => compile(job.body)); + } + assert.ok(project.workflows[0].jobs[0].body.includes(JSON.stringify(hostile).replace(/\u2028/g, "\\u2028"))); +}); + +test('compiled registration invokes the native metadata-selected create and emits minimal output', async () => { + const stub = await fixture(); + try { + const input = { submissionId: 'registration-001', localIdentifier: 'SYNTHETIC-FARM-001', name: 'Synthetic Farm' }; + const result = await runJob('registration', stub.baseUrl, input); + assert.equal(result.errors, undefined, JSON.stringify(result)); + assert.deepEqual(result.data, { status: 'registered', recordId: ID, traceId: TRACE }); + assertClean(result); + assert.equal(stub.requests.length, 2); + assert.equal(stub.requests[1].headers['idempotency-key'], 'registration:registration-001'); + assert.deepEqual(JSON.parse(stub.requests[1].body), { data: { localIdentifier: input.localIdentifier, name: input.name } }); + assert.ok(stub.requests.every(request => request.headers.authorization === 'Bearer synthetic-pilot-token')); + } finally { await stub.close(); } +}); + +test('compiled correction creates a draft, reads its action and submits with a separate idempotency key', async () => { + const stub = await fixture({ entity: 'name-correction' }); + try { + const result = await runJob('correction', stub.baseUrl, correctionInput); + assert.equal(result.errors, undefined, JSON.stringify(result)); + assert.deepEqual(result.data, { status: 'submitted', requestId: REQUEST_ID }); + assertClean(result); + const writes = stub.requests.filter(request => request.method === 'POST'); + assert.equal(writes.length, 2); + assert.deepEqual(JSON.parse(writes[0].body), { data: correctionData(correctionInput) }); + assert.equal(writes[0].headers['idempotency-key'], 'correction:correction-001'); + assert.equal(writes[1].headers['idempotency-key'], 'correction-submit:correction-001'); + assert.equal(writes[1].headers['if-match'], ACTION_ETAG); + assert.deepEqual(JSON.parse(writes[1].body), {}); + } finally { await stub.close(); } +}); + +test('same correction replay reconciles current state without duplicate lifecycle mutation', async () => { + const stub = await fixture({ entity: 'name-correction' }); + try { + const first = await runJob('correction', stub.baseUrl, correctionInput); + const replay = await runJob('correction', stub.baseUrl, correctionInput); + assert.equal(first.errors, undefined, JSON.stringify(first)); + assert.equal(replay.errors, undefined, JSON.stringify(replay)); + assert.deepEqual(replay.data, { status: 'already_submitted', requestId: REQUEST_ID }); + assert.equal(stub.mutationCount(), 2); + assert.equal(stub.requests.filter(request => request.url.includes('/actions/')).length, 1); + assertClean(replay); + } finally { await stub.close(); } +}); + +test('changed correction content is refused before already-submitted classification or action execution', async () => { + for (const currentState of ['draft', 'submitted']) { + const stub = await fixture({ entity: 'name-correction', currentState, currentData: { ...correctionData(correctionInput), name: 'A different correction' } }); + try { + const result = await runJob('correction', stub.baseUrl, correctionInput); + assert.ok(result.errors, JSON.stringify(result)); + assert.match(result.errors.pilot.message, /Correction content does not match/); + assert.equal(stub.requests.length, 3); + assert.equal(stub.requests.some(request => request.url.includes('/actions/')), false); + assertClean(result); + } finally { await stub.close(); } + } +}); + +test('unknown request states and native failures stop dependent mutations and retain safe failures', async () => { + for (const options of [{ entity: 'name-correction', currentState: 'cancelled' }, { entity: 'name-correction', failStatus: 403 }]) { + const stub = await fixture(options); + try { + const result = await runJob('correction', stub.baseUrl, correctionInput); + assert.ok(result.errors, JSON.stringify(result)); + assert.match(result.errors.pilot.message, options.failStatus ? /Registry operation did not complete: denied/ : /Correction is not ready for submission/); + assert.equal(stub.requests.some(request => request.url.includes('/actions/')), false); + assertClean(result); + } finally { await stub.close(); } + } +}); + +test('unstable submission identifiers fail before any HTTP exchange', async () => { + const stub = await fixture(); + try { + for (const submissionId of ['', 'event/with/path', undefined]) { + const result = await runJob('registration', stub.baseUrl, { submissionId }); + assert.ok(result.errors, JSON.stringify(result)); + assert.match(result.errors.pilot.message, /A stable submissionId is required/); + assertClean(result); + } + assert.equal(stub.requests.length, 0); + } finally { await stub.close(); } +}); + +test('registration replay reuses the durable key and a changed payload conflicts', async () => { + const stub = await fixture(); + try { + const input = { submissionId: 'registration-replay-001', localIdentifier: 'SYNTHETIC-FARM-001', name: 'Synthetic Farm' }; + const first = await runJob('registration', stub.baseUrl, input); + const replay = await runJob('registration', stub.baseUrl, input); + const changed = await runJob('registration', stub.baseUrl, { ...input, name: 'A different payload' }); + assert.equal(first.errors, undefined, JSON.stringify(first)); + assert.equal(replay.errors, undefined, JSON.stringify(replay)); + assert.deepEqual(replay.data, first.data); + assert.match(changed.errors.pilot.message, /Registry operation did not complete: conflict/); + assert.equal(stub.mutationCount(), 1); + assert.ok(stub.requests.filter(request => request.method === 'POST').every(request => request.headers['idempotency-key'] === 'registration:registration-replay-001')); + assertClean(replay); + assertClean(changed); + } finally { await stub.close(); } +}); + +test('a rejected correction replay returns its existing request without submission or mutation', async () => { + const stub = await fixture({ entity: 'name-correction', currentState: 'rejected', storedData: correctionData(correctionInput) }); + try { + for (let retry = 0; retry < 2; retry++) { + const result = await runJob('correction', stub.baseUrl, correctionInput); + assert.equal(result.errors, undefined, JSON.stringify(result)); + assert.deepEqual(result.data, { status: 'already_submitted', requestId: REQUEST_ID }); + assertClean(result); + } + assert.equal(stub.mutationCount(), 0); + assert.equal(stub.requests.some(request => request.url.includes('/actions/')), false); + const replays = stub.requests.filter(request => request.method === 'POST'); + assert.equal(replays.length, 2); + assert.ok(replays.every(request => request.headers['idempotency-key'] === 'correction:correction-001')); + for (const request of replays) assert.deepEqual(JSON.parse(request.body).data, correctionData(correctionInput)); + } finally { await stub.close(); } +}); diff --git a/pilot/test/provision.test.js b/pilot/test/provision.test.js new file mode 100644 index 0000000..3bf29e9 --- /dev/null +++ b/pilot/test/provision.test.js @@ -0,0 +1,234 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, readFile, readdir, rm, stat, unlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { privateFile, run } from '../lightning/provision.mjs'; +import { projectId, triggerIds } from '../lightning/project.mjs'; + +const readJson = async path => JSON.parse(await readFile(path, 'utf8')); +async function fixture(t) { + const root = await mkdtemp(join(tmpdir(), 'agriculture-provision-')); + t.after(() => rm(root, {recursive: true, force: true})); + const files = join(root, 'openfn'); + const secrets = join(files, 'secrets'); + await mkdir(secrets, {recursive: true}); + await mkdir(join(root, 'bridge')); + await writeFile(join(root, 'bridge', 'expected-source'), 'urn:synthetic:registry'); + await privateFile(join(secrets, 'api-token'), 'synthetic-test-token'); + await privateFile(join(secrets, 'destination-api-key'), 'synthetic-destination-key'); + await privateFile(join(files, 'breg-credential.json'), JSON.stringify({breg: {synthetic: 'private-breg'}})); + await privateFile(join(files, 'evidence-credential.json'), JSON.stringify({evidence: {profilePath: '/synthetic/profile.json'}})); + await privateFile(join(root, 'workflow-bindings.json'), JSON.stringify({ + breg: {accessProfile: 'synthetic', farmCreateOperation: 'records.farm.create', correctionCreateOperation: 'records.name-correction.create'}, + evidence: {requirement: 'holding-registered', registeredConcept: 'registered'}, + })); + return {root, files, secrets}; +} + +function remote({loseFirstCreate = false, credentials = []} = {}) { + const creations = []; + const provisioned = []; + const fetchImpl = async (input, options) => { + const url = new URL(input); + assert.equal(options.headers.authorization, 'Bearer synthetic-test-token'); + if (url.pathname === `/api/provision/${projectId}`) return new Response('{}'); + if (url.pathname === '/api/credentials' && options.method === 'GET') { + assert.equal(url.searchParams.get('project_id'), projectId); + return Response.json({credentials}); + } + if (url.pathname === '/api/credentials' && options.method === 'POST') { + const body = JSON.parse(options.body); + assert.equal(body.schema, 'raw'); + assert.deepEqual(Object.keys(body.credential_bodies[0]), ['name', 'body']); + assert.ok(!credentials.some(value => value.name === body.name), 'must not POST an already-created credential'); + creations.push(body); + const credential = {id: `credential-${creations.length}`, name: body.name, schema: 'raw', + project_credentials: [{project_id: projectId, id: `association-${creations.length}`}]}; + credentials.push(credential); + if (loseFirstCreate) { loseFirstCreate = false; throw new Error('synthetic response lost after commit'); } + return Response.json({credential}, {status: 201}); + } + if (url.pathname === '/api/provision' && options.method === 'POST') { + provisioned.push(JSON.parse(options.body)); + return Response.json({}); + } + assert.fail(`Unexpected fixture route ${options.method} ${url.pathname}`); + }; + return {fetchImpl, creations, credentials, provisioned}; +} + +test('credential creation survives lost POST response and resumes without duplicates', async t => { + const {root, secrets} = await fixture(t); + const api = remote({loseFirstCreate: true}); + await assert.rejects(run('provision', root, api), /response lost/); + const partial = await readJson(join(secrets, 'credentials.json')); + assert.equal(partial.credentialFormat, 2, 'format must be durable before the first POST'); + assert.ok(partial.credentialNames.breg); + assert.equal(partial.breg, undefined); + await run('provision', root, api); + const completed = await readJson(join(secrets, 'credentials.json')); + assert.equal(completed.breg, 'association-1'); + assert.equal(api.creations.length, 3); + assert.equal(api.provisioned.length, 1); + await run('provision', root, api); + assert.equal(api.creations.length, 3, 'repeat must preserve selected remote credentials'); +}); + +test('legacy associations remain untouched while replacement format uses a fresh namespace', async t => { + const {root, secrets} = await fixture(t); + const legacy = {breg: 'old-association'}; + await privateFile(join(secrets, 'credentials.json'), JSON.stringify(legacy)); + const old = {id: 'old', name: 'Agriculture breg main', schema: 'raw', project_credentials: [{project_id: projectId, id: 'old-association'}]}; + const api = remote({credentials: [old]}); + await run('provision', root, api); + const completed = await readJson(join(secrets, 'credentials.json')); + assert.deepEqual(completed.previous, legacy); + assert.equal(api.credentials[0], old); + assert.ok(api.creations.every(item => item.name !== old.name)); +}); + +test('prepare restores interrupted environment publication without rotating keys', async t => { + const {root, files, secrets} = await fixture(t); + await run('prepare', root); + const original = await readJson(join(secrets, 'environment.json')); + await unlink(join(files, 'worker.env')); + await unlink(join(files, 'lightning.env')); + await run('prepare', root); + for (const [name, value] of Object.entries(original)) { + assert.equal(await readFile(join(files, name), 'utf8'), value); + assert.equal((await stat(join(files, name))).mode & 0o777, 0o600); + } +}); + +test('prepare repairs legacy missing worker env using retained private key and database password', async t => { + const {root, files, secrets} = await fixture(t); + await run('prepare', root); + const original = await readJson(join(secrets, 'environment.json')); + await unlink(join(secrets, 'environment.json')); + await unlink(join(files, 'worker.env')); + await run('prepare', root); + for (const [name, value] of Object.entries(original)) assert.equal(await readFile(join(files, name), 'utf8'), value); +}); + +test('prepare refuses conflicting retained env files without replacing them', async t => { + const {root, files} = await fixture(t); + await run('prepare', root); + const changed = 'WORKER_SECRET=synthetic-conflict\n'; + await privateFile(join(files, 'worker.env'), changed); + await assert.rejects(run('prepare', root), /differs from its retained checkpoint/); + assert.equal(await readFile(join(files, 'worker.env'), 'utf8'), changed); +}); + +test('failed checkpoint write preserves prior bytes and removes staging file', async t => { + const {secrets} = await fixture(t); + const path = join(secrets, 'checkpoint.json'); + await privateFile(path, '{"complete":true}'); + await assert.rejects(privateFile(path, Symbol('not writable')), TypeError); + assert.equal(await readFile(path, 'utf8'), '{"complete":true}'); + assert.ok(!(await readdir(secrets)).some(name => name.endsWith('.tmp'))); +}); + +test('partial format-two checkpoint reuses the credential saved before the next request failed', async t => { + const {root, secrets} = await fixture(t); + const api = remote(); + let failSecond = true; + const interrupted = {fetchImpl: async (url, options) => { + if (new URL(url).pathname === '/api/credentials' && options.method === 'POST' && api.creations.length === 1 && failSecond) { + failSecond = false; + throw new Error('synthetic second credential unavailable'); + } + return api.fetchImpl(url, options); + }}; + await assert.rejects(run('provision', root, interrupted), /second credential unavailable/); + const before = await readJson(join(secrets, 'credentials.json')); + assert.equal(before.credentialFormat, 2); + assert.equal(before.breg, 'association-1'); + await run('provision', root, api); + assert.equal(api.creations.length, 3); + assert.equal((await readJson(join(secrets, 'credentials.json'))).breg, before.breg); +}); + +test('prepare gives the committed-event bridge a distinct retained key', async t => { + const {root, secrets} = await fixture(t); + await privateFile(join(secrets, 'webhook-api-key'), 'synthetic-retained-intake-key'); + await run('prepare', root); + const intake = await readFile(join(secrets, 'webhook-api-key'), 'utf8'); + const committed = await readFile(join(secrets, 'committed-api-key'), 'utf8'); + assert.equal(intake, 'synthetic-retained-intake-key'); + assert.match(committed, /^[0-9a-f]{64}$/); + assert.notEqual(committed, intake); + assert.equal(await readFile(join(root, 'bridge', 'openfn-api-key'), 'utf8'), committed); + await run('prepare', root); + assert.equal(await readFile(join(secrets, 'committed-api-key'), 'utf8'), committed); + assert.equal(await readFile(join(secrets, 'webhook-api-key'), 'utf8'), intake); +}); + +async function enableFixture(t, unsafeTrigger) { + const local = await fixture(t); + await privateFile(join(local.secrets, 'webhook-api-key'), 'synthetic-intake-key'); + await privateFile(join(local.secrets, 'committed-api-key'), 'synthetic-committed-key'); + await privateFile(join(local.secrets, 'credentials.json'), JSON.stringify({credentialFormat: 2})); + const probes = []; + let enabled = false; + const fetchImpl = async (input, options) => { + const path = new URL(input).pathname; + if (path.startsWith('/i/')) { + const trigger = path.slice(3); + const key = options.headers['x-api-key']; + assert.equal(options.method, 'GET', 'all auth checks must use the read-only route'); + assert.equal(options.body, undefined, 'no probe can enqueue a work order'); + const ownKey = trigger === triggerIds.committed ? 'synthetic-committed-key' : 'synthetic-intake-key'; + const isOwn = key === ownKey; + probes.push({trigger, key, isOwn}); + return new Response('{}', {status: key === undefined ? 401 : isOwn ? 200 : trigger === unsafeTrigger ? 200 : 404}); + } + assert.equal(path, '/api/provision'); + enabled = true; + const body = JSON.parse(options.body); + assert.ok(body.workflows.every(workflow => workflow.triggers.every(trigger => trigger.enabled))); + return Response.json({}); + }; + return {...local, fetchImpl, probes, enabled: () => enabled}; +} + +test('enable proves anonymous and cross-key rejection before enabling any workflow', async t => { + const fixture = await enableFixture(t); + await run('enable', fixture.root, fixture); + assert.equal(fixture.enabled(), true); + assert.deepEqual(fixture.probes.filter(probe => !probe.key).map(probe => probe.trigger).sort(), Object.values(triggerIds).sort()); + assert.equal(fixture.probes.filter(probe => probe.key && !probe.isOwn).length, 3); + assert.equal(fixture.probes.filter(probe => probe.isOwn).length, 3); +}); + +for (const [name, trigger] of Object.entries(triggerIds)) { + test(`enable refuses ${name} webhook accepting the other authority's key`, async t => { + const fixture = await enableFixture(t, trigger); + await assert.rejects(run('enable', fixture.root, fixture), /authority overlap/); + assert.equal(fixture.enabled(), false); + }); +} + +test('enable refuses missing intended-key access instead of accepting concealed nonexistence', async t => { + const fixture = await enableFixture(t); + const fetchImpl = async (url, options) => { + if (new URL(url).pathname === `/i/${triggerIds.committed}` && options.headers['x-api-key'] === 'synthetic-committed-key') { + return new Response('{}', {status: 404}); + } + return fixture.fetchImpl(url, options); + }; + await assert.rejects(run('enable', fixture.root, {fetchImpl}), /intended key is not accepted/); + assert.equal(fixture.enabled(), false); +}); + +test('enable refuses an anonymously accessible check route', async t => { + const fixture = await enableFixture(t); + const fetchImpl = async (url, options) => { + if (new URL(url).pathname.startsWith('/i/') && options.headers['x-api-key'] === undefined) { + return new Response('{}', {status: 200}); + } + return fixture.fetchImpl(url, options); + }; + await assert.rejects(run('enable', fixture.root, {fetchImpl}), /without authentication/); + assert.equal(fixture.enabled(), false); +}); diff --git a/pilot/test/recovery_test.py b/pilot/test/recovery_test.py new file mode 100644 index 0000000..c8178ff --- /dev/null +++ b/pilot/test/recovery_test.py @@ -0,0 +1,122 @@ +"""Offline guard and recovery checks for the live fault-injection driver.""" +import importlib.util +import json +from pathlib import Path +import subprocess +import sys +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +PILOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PILOT)) +SPEC = importlib.util.spec_from_file_location('recovery', PILOT / 'recovery.py') +recovery = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(recovery) +RECORD = '11111111-1111-4111-8111-111111111111' +EVENT = '22222222-2222-4222-8222-222222222222' + + +class RecoveryTests(unittest.TestCase): + def test_rpc_scope_and_sanitized_report(self): + report = {'workOrders': []} + with patch.object(recovery, 'captured', return_value='Unrelated console output\nPILOT_RECOVERY_JSON:' + json.dumps(report) + '\n:ok') as command: + self.assertEqual(recovery.rpc(RECORD), report) + self.assertIn('pilot_recovery_request: Base.url_decode64!', command.call_args.args[0][-1]) + self.assertNotIn('-e', command.call_args.args[0]) + with patch.object(recovery, 'captured') as command: + with self.assertRaises(recovery.SmokeFailure): + recovery.rpc("bad' OR true") + command.assert_not_called() + + def test_database_read_only_and_failure_output_withheld(self): + result = subprocess.CompletedProcess([], 0, stdout='BEGIN\n[]\nROLLBACK\n', stderr='') + with patch.object(recovery.subprocess, 'run', return_value=result) as command: + self.assertEqual(recovery.event_for_record(RECORD), []) + sql = command.call_args.kwargs['input'] + self.assertTrue(sql.startswith('BEGIN READ ONLY;')) + self.assertTrue(sql.endswith('ROLLBACK;\n')) + self.assertIn('-d agriculture', command.call_args.args[0][-1]) + result = subprocess.CompletedProcess([], 1, stdout='SYNTHETIC-SECRET-CANARY', stderr='SYNTHETIC-PAYLOAD-CANARY') + with patch.object(recovery.subprocess, 'run', return_value=result): + with self.assertRaises(recovery.SmokeFailure) as refusal: + recovery.delivery(EVENT) + self.assertNotIn('CANARY', str(refusal.exception)) + + def test_service_scope_disallows_database_or_volume_operations(self): + driver = recovery.Recovery.__new__(recovery.Recovery) + with patch.object(recovery, 'captured') as command: + for operation, services in [('down', ['worker']), ('stop', ['breg-db']), ('restart', ['mint'])]: + with self.assertRaises(recovery.SmokeFailure): + driver.service(operation, *services) + command.assert_not_called() + + def test_forwarding_failure_restores_lightning_even_if_create_fails(self): + driver = recovery.Recovery.__new__(recovery.Recovery) + scenario = {'phase': 'prepared'} + with patch.object(driver, 'scenario', return_value=scenario), patch.object(driver, 'service') as service, \ + patch.object(driver, 'ready') as ready, patch.object(driver, 'create', side_effect=recovery.SmokeFailure('injected')): + with self.assertRaises(recovery.SmokeFailure): + driver.before_accept() + self.assertEqual([call.args for call in service.call_args_list], [('stop', 'lightning'), ('start', 'lightning')]) + ready.assert_called_once() + + def test_downstream_failure_restores_destination_even_if_create_fails(self): + driver = recovery.Recovery.__new__(recovery.Recovery) + with patch.object(driver, 'scenario', return_value={'phase': 'prepared'}), patch.object(driver, 'service') as service, \ + patch.object(driver, 'destination_ready') as ready, patch.object(driver, 'create', side_effect=recovery.SmokeFailure('injected')): + with self.assertRaises(recovery.SmokeFailure): + driver.after_accept() + self.assertEqual([call.args for call in service.call_args_list], [('stop', 'destination'), ('start', 'destination')]) + ready.assert_called_once() + + def test_pending_failure_restores_worker(self): + driver = recovery.Recovery.__new__(recovery.Recovery) + with patch.object(driver, 'scenario', return_value={'phase': 'prepared'}), patch.object(driver, 'service') as service, \ + patch.object(driver, 'create', side_effect=recovery.SmokeFailure('injected')): + with self.assertRaises(recovery.SmokeFailure): + driver.pending_restart() + self.assertEqual([call.args for call in service.call_args_list], [('stop', 'worker'), ('start', 'worker')]) + + def test_retry_uses_existing_order_and_latest_failed_run_only(self): + driver = recovery.Recovery.__new__(recovery.Recovery) + orders = [{'id': 'order-1', 'state': 'failed', 'runs': [{'id': 'run-1'}, {'id': 'run-2'}]}, + {'id': 'order-2', 'state': 'success', 'runs': [{'id': 'run-3'}]}] + with patch.object(driver, 'orders', return_value=orders), patch.object(driver, 'poll'), \ + patch.object(recovery, 'rpc', return_value={'runId': 'run-4'}) as rpc: + self.assertEqual(driver.retry_failed(RECORD), [{'runId': 'run-4'}]) + rpc.assert_called_once_with(RECORD, 'retry', workOrderId='order-1', expectedRunId='run-2') + + def test_unproved_dead_letter_does_not_claim_complete(self): + driver = recovery.Recovery.__new__(recovery.Recovery) + scenario = {'phase': 'forwarding-failure', 'eventId': EVENT} + with patch.object(driver, 'scenario', return_value=scenario), \ + patch.object(recovery, 'delivery', return_value={'state': 'delivered'}), patch.object(driver, 'accepted') as accepted: + with self.assertRaises(recovery.SmokeFailure): + driver.before_accept() + accepted.assert_not_called() + + def test_smoke_replay_waits_for_success_and_preserves_existing_effect(self): + driver = recovery.Recovery.__new__(recovery.Recovery) + driver.pilot = SimpleNamespace(state={'recordId': RECORD}) + driver.state = {} + orders = [{'eventId': EVENT, 'state': 'success'}] + rows = {'effects': [{'event_id': EVENT}], 'records': [{'revision': 1}]} + dead = {'state': 'dead_lettered', 'payloadAvailable': True, 'generation': 1, + 'deliveryId': 'events.farm.farm-created-v1.webhook'} + settled = {'state': 'delivered', 'payloadAvailable': False, 'generation': 2} + with patch.object(driver, 'orders', return_value=orders), patch.object(driver, 'effects', return_value=rows), \ + patch.object(recovery, 'delivery', return_value=dead), patch.object(driver, 'poll', side_effect=[settled, orders]), \ + patch.object(recovery, 'captured') as command, patch.object(driver, 'save'): + self.assertEqual(driver.replay_smoke(), 1) + args = command.call_args.args[0] + self.assertIn(EVENT, args) + self.assertEqual(args[-2:], ['--expected-generation', '1']) + with patch.object(driver, 'orders', return_value=[{'state': 'pending'}]), patch.object(recovery, 'captured') as command: + with self.assertRaises(recovery.SmokeFailure): + driver.replay_smoke() + command.assert_not_called() + + +if __name__ == '__main__': + unittest.main() diff --git a/pilot/test/smoke_test.py b/pilot/test/smoke_test.py new file mode 100644 index 0000000..e80f5ec --- /dev/null +++ b/pilot/test/smoke_test.py @@ -0,0 +1,142 @@ +"""Offline proofs for smoke-driver recovery, review ordering, and safe diagnostics.""" +import importlib.util +import json +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +SPEC = importlib.util.spec_from_file_location('smoke', Path(__file__).resolve().parents[1] / 'smoke.py') +smoke = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(smoke) +RECORD = '00000000-0000-4000-8000-000000000001' +REQUEST = '00000000-0000-4000-8000-000000000002' + + +class FakePilot(smoke.Pilot): + def __init__(self, path): + self.state_path = path + self.timeout = 10 + self.state = {'schema': 'registry-openfn-smoke/v1', 'runId': 'synthetic-test', + 'registration': {'submissionId': 'registration-test', 'localIdentifier': 'TEST', 'name': 'Old Name'}, + 'correctedName': 'Correct Name', 'supportingReference': 'SYNTHETIC-TEST', 'reason': 'Correction', 'phase': 'prepared'} + self.events = 0 + self.calls = [] + self.current_name = None + self.request_state = None + + def destination(self): + return {'acceptedEvents': self.events + 100, 'appliedEffects': self.events + 100, 'records': int(self.events > 0) + 25} + + def webhook(self, name, data, expect_success=True): + self.calls.append(('webhook', name, expect_success)) + if name == 'registration': + if not expect_success: + assert self.current_name is not None + return None + if self.current_name is None: + self.current_name = data['name'] + self.events += 1 + return {'data': {'recordId': RECORD}} + if self.request_state is None: + self.request_state = 'submitted' + return {'data': {'requestId': REQUEST}} + + def farm(self): + return {'data': {'recordIdentifier': RECORD, 'domainData': {'name': self.current_name}}} + + def current_request(self): + return {'data': {'recordIdentifier': REQUEST, 'request': {'bregState': self.request_state}, + 'domainData': {'record': RECORD, 'name': self.state['correctedName'], 'reason': self.state['reason'], 'supportingReference': self.state['supportingReference']}}} + + def review(self, action): + self.calls.append(('review', action)) + if action == 'approve': + assert self.calls[-2] == ('review', 'inspect') + assert self.current_name == self.state['registration']['name'] + self.request_state = 'approved' + if action == 'apply': + assert self.calls[-2] == ('review', 'inspect') + assert self.request_state == 'approved' + self.request_state = 'applied' + self.current_name = self.state['correctedName'] + self.events += 1 + + def destination_rows(self): + return {'records': [{'registered': 1, 'record_id': RECORD, 'revision': 2}], + 'effects': [{'event_id': 'event1', 'outcome': 'applied'}, {'event_id': 'event2', 'outcome': 'applied'}][:self.events]} + + +class SmokeTests(unittest.TestCase): + def test_main_and_retained_replay_keep_review_order_and_event_count(self): + with tempfile.TemporaryDirectory() as directory: + pilot = FakePilot(Path(directory) / 'state.json') + pilot.run() + self.assertEqual(pilot.state['phase'], 'complete') + self.assertEqual(pilot.events, 2) + self.assertEqual([call for call in pilot.calls if call[0] == 'review'], + [('review', 'inspect'), ('review', 'approve'), ('review', 'inspect'), ('review', 'apply')]) + pilot.run() + self.assertEqual(pilot.events, 2) + self.assertEqual(len([call for call in pilot.calls if call == ('review', 'apply')]), 1) + self.assertEqual(pilot.state_path.stat().st_mode & 0o777, 0o600) + + def test_partial_approval_recovers_without_approving_again(self): + with tempfile.TemporaryDirectory() as directory: + pilot = FakePilot(Path(directory) / 'state.json') + pilot.state.update(recordId=RECORD, requestId=REQUEST, baseline={'acceptedEvents': 0}, phase='approved') + pilot.current_name = 'Old Name' + pilot.request_state = 'approved' + pilot.events = 1 + pilot.run() + self.assertEqual(pilot.events, 2) + self.assertNotIn(('review', 'approve'), pilot.calls) + self.assertIn(('review', 'apply'), pilot.calls) + + def test_restart_requires_completed_main_and_only_restarts_retained_services(self): + with tempfile.TemporaryDirectory() as directory: + pilot = FakePilot(Path(directory) / 'state.json') + with patch.object(smoke, 'captured') as command: + with self.assertRaises(smoke.SmokeFailure): + pilot.restart_recovery() + command.assert_not_called() + pilot.run() + with patch.object(smoke, 'captured', return_value='') as command, patch.object(smoke, 'http_json', return_value=(200, None)): + pilot.restart_recovery() + self.assertEqual(command.call_args.args[0][-6:], ['restart', 'breg', 'worker', 'bridge', 'destination', 'lightning']) + self.assertEqual(pilot.state['restartRecovery'], 'passed') + self.assertEqual(pilot.events, 2) + + def test_subprocess_failures_withhold_secret_outputs(self): + result = subprocess.CompletedProcess(['synthetic'], 1, stdout='secret-token-canary', stderr='secret-record-canary') + with patch.object(smoke.subprocess, 'run', return_value=result): + with self.assertRaises(smoke.SmokeFailure) as refusal: + smoke.captured(['synthetic'], 'Synthetic command') + self.assertNotIn('canary', str(refusal.exception)) + + def test_metadata_path_rejects_foreign_or_unsubstituted_routes(self): + self.assertEqual(smoke.operation_path({'path': '/v1/records/farms/{record_id}'}, RECORD), '/v1/records/farms/' + RECORD) + for path in ['https://attacker.invalid/write', '/v1/../secret', '/v1/records/{other}', '/v1/records?token=secret']: + with self.assertRaises(smoke.SmokeFailure): + smoke.operation_path({'path': path}) + + def test_request_id_recovery_accepts_sync_output_without_replaying(self): + with tempfile.TemporaryDirectory() as directory: + pilot = FakePilot(Path(directory) / 'state.json') + for output in [{'data': {'requestId': REQUEST}}, {'data': {'data': {'requestId': REQUEST}}}]: + with patch.object(pilot, 'metadata', side_effect=AssertionError('Unexpected replay')): + self.assertEqual(pilot.recover_request_id(output), REQUEST) + + def test_changed_current_request_refuses_review(self): + with tempfile.TemporaryDirectory() as directory: + pilot = FakePilot(Path(directory) / 'state.json') + pilot.state['recordId'] = RECORD + current = pilot.current_request() + current['data']['domainData']['supportingReference'] = 'other-submission' + with self.assertRaises(smoke.SmokeFailure): + pilot.inspect_content(current) + + +if __name__ == '__main__': + unittest.main() diff --git a/services/destination/README.md b/services/destination/README.md new file mode 100644 index 0000000..f9b9bd8 --- /dev/null +++ b/services/destination/README.md @@ -0,0 +1,64 @@ +# Synthetic destination + +This pilot destination persists the minimized `registered` result in SQLite. +It uses Node.js 22.23.2 built-ins, including experimental `node:sqlite`, with no +package dependencies. Start with `node services/destination/src/server.js` from +the repository root. + +Set `DESTINATION_API_KEY_FILE` to a file containing a printable secret of at +least 32 bytes, with no trailing newline. Set `DESTINATION_DB_PATH` to an +absolute database path on a persistent writable volume. `PORT` defaults to +8082. The process creates the parent directory with mode 0700 and uses umask +0077. Mount existing volumes with ownership suitable for the runtime user. +Keep the SQLite file and WAL sidecars together; restarting the service must +reuse this volume. There is no reset or delete endpoint. + +`POST /updates` requires `Content-Type: application/json` and +`x-destination-key: `. Its body is bounded to 16384 bytes and accepts +exactly these fields: + +```json +{ + "source": "urn:registrystack:registry:agricultural-holdings:instance:agriculture-openfn-pilot", + "eventId": "22222222-2222-4222-8222-222222222222", + "effect": "record-sync", + "recordId": "11111111-1111-4111-8111-111111111111", + "revision": 1, + "values": { "registered": true } +} +``` + +No local identifier, owner name, raw Evidence response, token, delivery +generation, or signed assertion belongs in this minimized update. `registered` +is a boolean; the workflow is responsible for deriving it from its verified +Evidence result. The destination trusts its authenticated workflow caller and +does not independently verify Evidence signatures. + +SQLite's primary key is `(source, eventId, effect)`. The effect ledger and +record update share one `BEGIN IMMEDIATE` transaction, with WAL journaling and +`synchronous=FULL`. A changed payload under the same effect key returns 409. +Records are keyed by `(source, recordId, effect)` and advance only to a higher +revision. Lower revisions and equal revisions with equal values are recorded +as stale; conflicting values at the same revision return 409. + +Successful responses use HTTP 200 and `{ "status": "applied", "revision": 1 }`. +The status is `duplicate` for an already recorded identical event, or `stale` +when its revision cannot advance the record. A duplicate response reports the +event's original revision; stale responses report the current record revision. +Do not infer a second effect from a successful retry. Losing a response after +commit and then retrying, including after restart, returns a duplicate while +preserving one effect. Unavailable storage returns 503 and no success is +released before commit. + +`GET /status` requires the same secret header and returns only +`acceptedEvents`, `appliedEffects`, `staleEvents`, `records`, and sorted numeric +`revisions`. It omits source values, identifiers, and payloads. `GET /healthz` +returns a static response after the database opens successfully. The service +does not log secrets or request payloads. + +This store makes its own synthetic record update atomic. To replace it with a +real external effect, that destination must provide its own atomic +deduplication contract; placing an HTTP send next to this SQLite transaction +would not make the remote effect exactly once. Retain the ledger for the +required event replay horizon and back up its volume using SQLite-aware +operations. Run checks with `npm --prefix services/destination test`. diff --git a/services/destination/package.json b/services/destination/package.json new file mode 100644 index 0000000..e6204df --- /dev/null +++ b/services/destination/package.json @@ -0,0 +1,12 @@ +{ + "name": "registry-pilot-destination", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "start": "node src/server.js", + "check": "node --check src/server.js", + "test": "node --test test/*.test.js" + }, + "engines": { "node": ">=22.23.2" } +} diff --git a/services/destination/src/server.js b/services/destination/src/server.js new file mode 100644 index 0000000..0c1db80 --- /dev/null +++ b/services/destination/src/server.js @@ -0,0 +1,185 @@ +import { createHash, timingSafeEqual } from 'node:crypto'; +import { readFileSync, mkdirSync } from 'node:fs'; +import http from 'node:http'; +import { dirname, isAbsolute } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +import { pathToFileURL } from 'node:url'; + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; +const object = value => value !== null && typeof value === 'object' && !Array.isArray(value); +class Refusal extends Error { + constructor(status, code) { super(code); this.status = status; } +} + +export function loadConfig(env = process.env) { + try { + const apiKey = readFileSync(env.DESTINATION_API_KEY_FILE, 'utf8'); + const databasePath = env.DESTINATION_DB_PATH; + const port = Number(env.PORT ?? 8082); + if (!/^[\x21-\x7e]{32,}$/.test(apiKey) || !databasePath || !isAbsolute(databasePath) || + !Number.isSafeInteger(port) || port < 1 || port > 65535) throw new Error(); + return { apiKey, databasePath, port }; + } catch { throw new Error('invalid destination configuration'); } +} + +function validate(update) { + const fields = ['source', 'eventId', 'effect', 'recordId', 'revision', 'values']; + if (!object(update) || Object.keys(update).length !== fields.length || fields.some(key => !Object.hasOwn(update, key)) || + typeof update.source !== 'string' || !update.source.startsWith('urn:registrystack:registry:') || + Buffer.byteLength(update.source) > 512 || typeof update.eventId !== 'string' || !UUID.test(update.eventId) || + typeof update.recordId !== 'string' || !UUID.test(update.recordId) || + update.effect !== 'record-sync' || !Number.isSafeInteger(update.revision) || update.revision < 1 || + !object(update.values) || Object.keys(update.values).length !== 1 || typeof update.values.registered !== 'boolean') { + throw new Refusal(400, 'invalid_update'); + } + return update; +} + +export class DestinationStore { + constructor(databasePath) { + if (databasePath !== ':memory:') mkdirSync(dirname(databasePath), { recursive: true, mode: 0o700 }); + this.database = new DatabaseSync(databasePath); + this.database.exec(` + PRAGMA busy_timeout = 5000; + PRAGMA journal_mode = WAL; + PRAGMA synchronous = FULL; + PRAGMA foreign_keys = ON; + CREATE TABLE IF NOT EXISTS effects ( + source TEXT NOT NULL, event_id TEXT NOT NULL, effect TEXT NOT NULL, + payload_hash TEXT NOT NULL, record_id TEXT NOT NULL, revision INTEGER NOT NULL CHECK(revision > 0), + outcome TEXT NOT NULL CHECK(outcome IN ('applied', 'stale')), + PRIMARY KEY(source, event_id, effect) + ) STRICT; + CREATE TABLE IF NOT EXISTS records ( + source TEXT NOT NULL, record_id TEXT NOT NULL, effect TEXT NOT NULL, + revision INTEGER NOT NULL CHECK(revision > 0), registered INTEGER NOT NULL CHECK(registered IN (0, 1)), + PRIMARY KEY(source, record_id, effect) + ) STRICT; + `); + } + + apply(input) { + const update = validate(input); + // Fixed ordered fields make key order irrelevant without changing values. + const payloadHash = createHash('sha256').update(JSON.stringify([ + update.source, update.eventId, update.effect, update.recordId, update.revision, update.values.registered, + ])).digest('hex'); + const db = this.database; + db.exec('BEGIN IMMEDIATE'); + try { + const previous = db.prepare('SELECT payload_hash, revision FROM effects WHERE source = ? AND event_id = ? AND effect = ?') + .get(update.source, update.eventId, update.effect); + if (previous) { + if (previous.payload_hash !== payloadHash) throw new Refusal(409, 'effect_conflict'); + db.exec('COMMIT'); + return { status: 'duplicate', revision: previous.revision }; + } + const record = db.prepare('SELECT revision, registered FROM records WHERE source = ? AND record_id = ? AND effect = ?') + .get(update.source, update.recordId, update.effect); + // Two distinct events cannot silently assign different values to the same revision. + if (record && record.revision === update.revision && record.registered !== Number(update.values.registered)) { + throw new Refusal(409, 'revision_conflict'); + } + const status = record && record.revision >= update.revision ? 'stale' : 'applied'; + if (status === 'applied') { + db.prepare(`INSERT INTO records (source, record_id, effect, revision, registered) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(source, record_id, effect) DO UPDATE SET revision = excluded.revision, registered = excluded.registered`) + .run(update.source, update.recordId, update.effect, update.revision, Number(update.values.registered)); + } + db.prepare('INSERT INTO effects (source, event_id, effect, payload_hash, record_id, revision, outcome) VALUES (?, ?, ?, ?, ?, ?, ?)') + .run(update.source, update.eventId, update.effect, payloadHash, update.recordId, update.revision, status); + db.exec('COMMIT'); + return { status, revision: Math.max(record?.revision ?? 0, update.revision) }; + } catch (error) { + db.exec('ROLLBACK'); + throw error; + } + } + + status() { + const counts = this.database.prepare(`SELECT count(*) AS acceptedEvents, + coalesce(sum(outcome = 'applied'), 0) AS appliedEffects, + coalesce(sum(outcome = 'stale'), 0) AS staleEvents FROM effects`).get(); + const records = this.database.prepare('SELECT revision FROM records ORDER BY revision').all(); + // No record ids, source values, evidence, or payloads are exposed by diagnostics. + return { ...counts, records: records.length, revisions: records.map(record => record.revision) }; + } + + close() { this.database.close(); } +} + +function authenticated(request, key) { + const count = request.rawHeaders.filter((value, index) => index % 2 === 0 && value.toLowerCase() === 'x-destination-key').length; + const provided = request.headers['x-destination-key']; + if (count !== 1 || typeof provided !== 'string') return false; + // Fixed-size hashes avoid an input-controlled timingSafeEqual length exception. + return timingSafeEqual(createHash('sha256').update(provided).digest(), createHash('sha256').update(key).digest()); +} + +function readUpdate(request) { + return new Promise((resolve, reject) => { + const chunks = []; + let size = 0; + let done = false; + const finish = (error, value) => { + if (done) return; + done = true; + clearTimeout(timer); + error ? reject(error) : resolve(value); + }; + const timer = setTimeout(() => finish(new Refusal(408, 'request_timeout')), 5000); + request.on('data', chunk => { + if (done) return; + size += chunk.length; + if (size > 16384) finish(new Refusal(413, 'body_too_large')); + else chunks.push(chunk); + }); + request.on('end', () => { + try { finish(null, JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks)))); } + catch { finish(new Refusal(400, 'invalid_update')); } + }); + request.on('error', () => finish(new Refusal(400, 'invalid_update'))); + }); +} + +function respond(response, status, body) { + if (!response.destroyed && !response.writableEnded) { + response.writeHead(status, { 'content-type': 'application/json', 'cache-control': 'no-store', + ...(status >= 400 ? { connection: 'close' } : {}) }); + response.end(JSON.stringify(body)); + } +} + +export function createDestination(config, store = new DestinationStore(config.databasePath)) { + const server = http.createServer({ maxHeaderSize: 8192 }, async (request, response) => { + try { + if (request.url === '/healthz' && request.method === 'GET') return respond(response, 200, { status: 'ok' }); + if (!authenticated(request, config.apiKey)) throw new Refusal(401, 'unauthorized'); + if (request.url === '/status' && request.method === 'GET') return respond(response, 200, store.status()); + if (request.url !== '/updates' || request.method !== 'POST') throw new Refusal(404, 'not_found'); + if (request.headers['content-type'] !== 'application/json' || request.headers['content-encoding'] !== undefined) { + throw new Refusal(400, 'invalid_update'); + } + const result = store.apply(await readUpdate(request)); + respond(response, 200, result); + } catch (error) { + respond(response, error instanceof Refusal ? error.status : 503, + { status: error instanceof Refusal ? error.message : 'unavailable' }); + } + }); + server.requestTimeout = 10000; + server.headersTimeout = 5000; + server.on('close', () => store.close()); + return server; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + process.umask(0o077); + const config = loadConfig(); + const server = createDestination(config); + server.on('error', () => { process.stderr.write('destination unavailable\n'); process.exitCode = 1; }); + server.listen(config.port, '0.0.0.0'); + for (const signal of ['SIGINT', 'SIGTERM']) process.on(signal, () => server.close()); + } catch { process.stderr.write('invalid destination configuration\n'); process.exitCode = 1; } +} diff --git a/services/destination/test/destination.test.js b/services/destination/test/destination.test.js new file mode 100644 index 0000000..8c75d1e --- /dev/null +++ b/services/destination/test/destination.test.js @@ -0,0 +1,142 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { test } from 'node:test'; +import { Worker } from 'node:worker_threads'; +import { createDestination, DestinationStore } from '../src/server.js'; + +const apiKey = 'synthetic-destination-key-at-least-32-bytes'; +const input = { source: 'urn:registrystack:registry:agricultural-holdings:instance:pilot', + eventId: '22222222-2222-4222-8222-222222222222', effect: 'record-sync', + recordId: '11111111-1111-4111-8111-111111111111', revision: 1, values: { registered: true } }; +const event = number => `22222222-2222-4222-8222-${String(number).padStart(12, '0')}`; + +async function directory(t) { + const path = await mkdtemp(join(tmpdir(), 'destination-test-')); + t.after(() => rm(path, { recursive: true, force: true })); + return join(path, 'state.sqlite'); +} +async function listen(server) { + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + return `http://127.0.0.1:${server.address().port}`; +} +async function close(server) { + server.closeAllConnections(); + await new Promise(resolve => server.close(resolve)); +} +const post = (url, value = input, key = apiKey) => fetch(`${url}/updates`, { method: 'POST', + headers: { 'content-type': 'application/json', 'x-destination-key': key }, body: JSON.stringify(value) }); + +test('durable dedupe, stale revision rejection, and restart retain exactly one effect', async t => { + const path = await directory(t); + let store = new DestinationStore(path); + assert.deepEqual(store.apply(input), { status: 'applied', revision: 1 }); + assert.deepEqual(store.apply(input), { status: 'duplicate', revision: 1 }); + assert.deepEqual(store.apply({ ...input, eventId: event(3), revision: 3, values: { registered: false } }), { status: 'applied', revision: 3 }); + assert.deepEqual(store.apply({ ...input, eventId: event(2), revision: 2 }), { status: 'stale', revision: 3 }); + assert.deepEqual(store.status(), { acceptedEvents: 3, appliedEffects: 2, staleEvents: 1, records: 1, revisions: [3] }); + store.close(); + store = new DestinationStore(path); + assert.equal(store.apply(input).status, 'duplicate'); + assert.equal(store.apply({ ...input, eventId: event(2), revision: 2 }).status, 'duplicate'); + assert.deepEqual(store.status(), { acceptedEvents: 3, appliedEffects: 2, staleEvents: 1, records: 1, revisions: [3] }); + store.close(); +}); + +test('same effect key with changed payload and equal revision with conflicting value refuse atomically', async t => { + const store = new DestinationStore(await directory(t)); + t.after(() => store.close()); + store.apply(input); + for (const changed of [{ ...input, revision: 2 }, { ...input, values: { registered: false } }, + { ...input, recordId: '99999999-9999-4999-8999-999999999999' }]) { + assert.throws(() => store.apply(changed), { message: 'effect_conflict', status: 409 }); + } + assert.throws(() => store.apply({ ...input, eventId: event(2), values: { registered: false } }), + { message: 'revision_conflict', status: 409 }); + assert.deepEqual(store.status(), { acceptedEvents: 1, appliedEffects: 1, staleEvents: 0, records: 1, revisions: [1] }); +}); + +test('an effect ledger failure rolls back its record update', async t => { + const store = new DestinationStore(await directory(t)); + t.after(() => store.close()); + store.database.exec("CREATE TRIGGER fail_effect BEFORE INSERT ON effects BEGIN SELECT RAISE(ABORT, 'injected'); END"); + assert.throws(() => store.apply(input)); + assert.deepEqual(store.status(), { acceptedEvents: 0, appliedEffects: 0, staleEvents: 0, records: 0, revisions: [] }); + store.database.exec('DROP TRIGGER fail_effect'); + assert.equal(store.apply(input).status, 'applied'); +}); + +test('concurrent independent SQLite writers atomically deduplicate one effect', async t => { + const path = await directory(t); + const initial = new DestinationStore(path); + initial.close(); + const moduleUrl = new URL('../src/server.js', import.meta.url).href; + const results = await Promise.all(Array.from({ length: 6 }, () => new Promise((resolve, reject) => { + const worker = new Worker(` + const { parentPort, workerData } = require('node:worker_threads'); + import(workerData.moduleUrl).then(({ DestinationStore }) => { + const store = new DestinationStore(workerData.path); + const result = store.apply(workerData.input); + store.close(); parentPort.postMessage(result); + }).catch(error => { throw error; }); + `, { eval: true, workerData: { moduleUrl, path, input } }); + worker.once('message', resolve); worker.once('error', reject); + }))); + assert.equal(results.filter(result => result.status === 'applied').length, 1); + assert.equal(results.filter(result => result.status === 'duplicate').length, 5); + const store = new DestinationStore(path); + assert.equal(store.status().appliedEffects, 1); + store.close(); +}); + +test('HTTP authentication, minimized payload, concurrent retries, and safe diagnostics', async t => { + const server = createDestination({ databasePath: await directory(t), apiKey }); + const url = await listen(server); + t.after(() => close(server)); + assert.equal((await post(url, input, 'wrong-secret')).status, 401); + assert.equal((await fetch(`${url}/status`)).status, 401); + const rejected = await post(url, { ...input, values: { registered: true, ownerName: 'SYNTHETIC-PRIVATE-CANARY' } }); + assert.equal(rejected.status, 400); + assert.equal((await rejected.text()).includes('SYNTHETIC-PRIVATE-CANARY'), false); + const results = await Promise.all(Array.from({ length: 12 }, async () => (await post(url)).json())); + assert.equal(results.filter(result => result.status === 'applied').length, 1); + assert.equal(results.filter(result => result.status === 'duplicate').length, 11); + const status = await fetch(`${url}/status`, { headers: { 'x-destination-key': apiKey } }); + assert.deepEqual(await status.json(), { acceptedEvents: 1, appliedEffects: 1, staleEvents: 0, records: 1, revisions: [1] }); + assert.equal((await post(url, { ...input, extra: 'x'.repeat(17000) })).status, 413); +}); + +test('lost response after committed effect is safe on workflow retry and service restart', async t => { + const path = await directory(t); + const store = new DestinationStore(path); + const originalApply = store.apply.bind(store); + let server; + store.apply = update => { + const result = originalApply(update); + // Deterministically drop the TCP connection after commit, before any reply. + server.closeAllConnections(); + return result; + }; + server = createDestination({ apiKey, databasePath: path }, store); + const url = await listen(server); + await assert.rejects(post(url)); + await close(server); + const restarted = createDestination({ apiKey, databasePath: path }); + const restartedUrl = await listen(restarted); + t.after(() => close(restarted)); + assert.deepEqual(await (await post(restartedUrl)).json(), { status: 'duplicate', revision: 1 }); + const status = await fetch(`${restartedUrl}/status`, { headers: { 'x-destination-key': apiKey } }); + assert.equal((await status.json()).appliedEffects, 1); +}); + +test('generation is not an effect identity field and source scopes record revisions', async t => { + const store = new DestinationStore(await directory(t)); + t.after(() => store.close()); + assert.throws(() => store.apply({ ...input, generation: 2 }), { message: 'invalid_update' }); + assert.throws(() => store.apply({ ...input, eventId: [input.eventId] }), { message: 'invalid_update' }); + assert.throws(() => store.apply({ ...input, recordId: [input.recordId] }), { message: 'invalid_update' }); + store.apply({ ...input, revision: 4 }); + assert.equal(store.apply({ ...input, source: `${input.source}-other` }).status, 'applied'); + assert.equal(store.status().records, 2); +}); diff --git a/services/event-bridge/README.md b/services/event-bridge/README.md new file mode 100644 index 0000000..1c05a9a --- /dev/null +++ b/services/event-bridge/README.md @@ -0,0 +1,88 @@ +# BREG event bridge + +This pilot service verifies BREG CloudEvents before submitting a work order to a +fixed OpenFn webhook. It uses Node.js 22.23.2 built-ins and has no package +dependencies. Start with `node services/event-bridge/src/server.js` from the +repository root. + +`POST /events/breg` is the only ingress route. The method, path without query, +content type, all signed headers, and bounded exact body bytes are covered by +the BREG `breg-webhook-signature-v1` HMAC contract: unsigned 64-bit big-endian +byte lengths precede each field in the order defined in Registry Stack's +`reference/breg-api` signature section. Secret bytes are neither trimmed nor +decoded. Signatures are compared in constant time. Duplicate signed headers, +content encoding, wrong source/type/schema, stale delivery times, and invalid +payloads are refused before forwarding. + +The pilot accepts exactly the common BREG payload members `entity`, `recordId`, +`revision`, `trigger`, `packageRevision`, and `values`. Entity, event schema, +trigger, and projected field names must match configuration. Record/event ids +must be canonical lowercase UUIDs, revision a positive safe integer, package +revision a SHA-256 binding, and projected values strings of at most 1024 UTF-8 +bytes. This deliberately implements the pilot's string projection; it is not +a general JSON Schema validator or lifecycle-event receiver. + +Configuration is required unless a default is listed: + +| Variable | Meaning | +| --- | --- | +| `BREG_HMAC_KEY_FILE` | File containing exact HMAC key bytes, at least 32 bytes. | +| `BREG_EXPECTED_SOURCE` | Exact CloudEvent source for the registry instance. | +| `BREG_EXPECTED_ENTITY` | Exact payload entity, `farm` for this pilot. | +| `BREG_EXPECTED_EVENTS_FILE` | JSON object mapping each exact type to `{ "schema": "exact-dataschema", "trigger": "created" }`; `patched` is also supported. | +| `BREG_ALLOWED_VALUE_FIELDS_FILE` | JSON array of exact projected string fields, `["local-identifier"]` for this pilot. | +| `OPENFN_WEBHOOK_URL` | Fixed webhook URL, with no credentials, query or fragment. | +| `OPENFN_API_KEY_FILE` | Exact printable API key for `x-api-key`; no trailing newline. | +| `ALLOW_HTTP` | Default false. Set exactly `true` only for an explicitly trusted internal HTTP deployment. HTTPS otherwise required. | +| `PORT` | Default 8081. | +| `MAX_BODY_BYTES` | Default 65536; maximum 1048576. | +| `MAX_DELIVERY_SKEW_SECONDS` | Default 300; maximum 3600. Applies to delivery time, not original event time. | +| `OPENFN_TIMEOUT_MS` | Default 3000; maximum 30000. Set below BREG's configured attempt timeout. | + +The authenticated OpenFn webhook receives this JSON envelope: + +```json +{ + "event": { + "specversion": "1.0", + "id": "22222222-2222-4222-8222-222222222222", + "source": "urn:registrystack:registry:agricultural-holdings:instance:agriculture-openfn-pilot", + "type": "farm-created-v1", + "time": "2026-09-09T00:00:00Z", + "dataschema": "exact configured schema binding" + }, + "delivery": { + "generation": 1, + "attempt": 1, + "time": "2026-09-09T00:00:01Z", + "idempotencyKey": "sha256:<64 lowercase hex characters>" + }, + "data": { + "entity": "farm", + "recordId": "11111111-1111-4111-8111-111111111111", + "revision": 1, + "trigger": "created", + "packageRevision": "sha256:<64 lowercase hex characters>", + "values": { "local-identifier": "SYNTHETIC-FARM-001" } + } +} +``` + +A 2xx OpenFn response must contain a UUID `work_order_id` and no `error` field. +Lightning 2.18.2 does not return a `run_id` for asynchronous acceptance. +Only after this check does the bridge return `202 {"status":"accepted"}` to BREG. This +acknowledges work-order acceptance, not workflow completion. Upstream rejection, +malformed/generic success responses, redirects, transport failures, or oversized +responses produce 502; deadline expiry produces 504. There are no internal +retries, redirect following, or persistent bridge inbox. + +If acceptance occurs but the response is lost, BREG can submit another work +order. The workflow and destination must tolerate this, including replay under +a new generation. Destination deduplication uses `(source, eventId, effect)`, +never the generation-specific delivery idempotency key. OpenFn's direct webhook +must remain API-key protected and accessible only to this trusted bridge and +administrators because it contains no independent BREG verifier. + +`GET /healthz` returns a static readiness response. Request payloads, headers, +secrets, and upstream error bodies are never logged or reflected in failures. +Run focused checks with `npm --prefix services/event-bridge test`. diff --git a/services/event-bridge/package.json b/services/event-bridge/package.json new file mode 100644 index 0000000..2c0c536 --- /dev/null +++ b/services/event-bridge/package.json @@ -0,0 +1,12 @@ +{ + "name": "registry-event-bridge", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "start": "node src/server.js", + "check": "node --check src/server.js", + "test": "node --test test/*.test.js" + }, + "engines": { "node": ">=22.23.2" } +} diff --git a/services/event-bridge/src/server.js b/services/event-bridge/src/server.js new file mode 100644 index 0000000..7617b10 --- /dev/null +++ b/services/event-bridge/src/server.js @@ -0,0 +1,217 @@ +import { createHmac, timingSafeEqual } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import http from 'node:http'; +import https from 'node:https'; +import { pathToFileURL } from 'node:url'; + +export const EVENT_PATH = '/events/breg'; +const SIGNED_HEADERS = [ + 'ce-specversion', 'ce-id', 'ce-source', 'ce-type', 'ce-time', 'ce-dataschema', + 'x-registry-event-generation', 'x-registry-delivery-attempt', 'x-registry-delivery-time', +]; +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; +const DIGEST = /^sha256:[0-9a-f]{64}$/; +const DATE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?(?:Z|[+-]\d{2}:\d{2})$/; +const object = value => value !== null && typeof value === 'object' && !Array.isArray(value); + +class Refusal extends Error { + constructor(status, code) { super(code); this.status = status; } +} + +function positive(value, fallback, maximum) { + const number = value === undefined ? fallback : Number(value); + if (!Number.isSafeInteger(number) || number < 1 || number > maximum) throw new Error('invalid configuration'); + return number; +} + +export function loadConfig(env = process.env) { + try { + const hmacKey = readFileSync(env.BREG_HMAC_KEY_FILE); + const apiKey = readFileSync(env.OPENFN_API_KEY_FILE, 'utf8'); + const expectedEvents = JSON.parse(readFileSync(env.BREG_EXPECTED_EVENTS_FILE, 'utf8')); + const allowedValueFields = JSON.parse(readFileSync(env.BREG_ALLOWED_VALUE_FIELDS_FILE, 'utf8')); + const url = new URL(env.OPENFN_WEBHOOK_URL); + if (hmacKey.length < 32 || !/^[\x21-\x7e]+$/.test(apiKey) || + !env.BREG_EXPECTED_SOURCE || !env.BREG_EXPECTED_ENTITY || + !object(expectedEvents) || Object.keys(expectedEvents).length === 0 || + Object.entries(expectedEvents).some(([type, binding]) => !type || !object(binding) || + typeof binding.schema !== 'string' || !binding.schema || !['created', 'patched'].includes(binding.trigger)) || + !Array.isArray(allowedValueFields) || allowedValueFields.length === 0 || + allowedValueFields.some(field => typeof field !== 'string' || !field) || + new Set(allowedValueFields).size !== allowedValueFields.length || + url.username || url.password || url.hash || url.search || + (url.protocol !== 'https:' && !(url.protocol === 'http:' && env.ALLOW_HTTP === 'true'))) { + throw new Error('invalid configuration'); + } + return { + hmacKey, apiKey, expectedEvents, allowedValueFields, + expectedSource: env.BREG_EXPECTED_SOURCE, expectedEntity: env.BREG_EXPECTED_ENTITY, + openfnUrl: url, port: positive(env.PORT, 8081, 65535), + maxBodyBytes: positive(env.MAX_BODY_BYTES, 65536, 1048576), + maxDeliverySkewSeconds: positive(env.MAX_DELIVERY_SKEW_SECONDS, 300, 3600), + timeoutMs: positive(env.OPENFN_TIMEOUT_MS, 3000, 30000), + }; + } catch { + // File paths and parser errors can include operator secrets or input values. + throw new Error('invalid bridge configuration'); + } +} + +export function webhookSignature(key, headers, body, method = 'POST', path = EVENT_PATH) { + const mac = createHmac('sha256', key).update('breg-webhook-signature-v1', 'ascii'); + const values = [...SIGNED_HEADERS.map(name => Buffer.from(headers[name], 'latin1')), + Buffer.from(method), Buffer.from(path), Buffer.from(headers['content-type'], 'latin1'), + Buffer.from(headers['idempotency-key'], 'latin1'), body]; + for (const value of values) { + const length = Buffer.alloc(8); + length.writeBigUInt64BE(BigInt(value.length)); + mac.update(length).update(value); + } + return `v1=${mac.digest('base64url')}`; +} + +function verifyHeaders(request, config) { + const required = [...SIGNED_HEADERS, 'content-type', 'idempotency-key', 'x-registry-signature']; + for (const name of required) { + const matches = request.rawHeaders.filter((value, index) => index % 2 === 0 && value.toLowerCase() === name); + if (matches.length !== 1 || typeof request.headers[name] !== 'string' || !request.headers[name]) { + throw new Refusal(400, 'invalid_event'); + } + } + const h = request.headers; + const binding = Object.hasOwn(config.expectedEvents, h['ce-type']) ? config.expectedEvents[h['ce-type']] : undefined; + const schema = binding?.schema; + const delivered = Date.parse(h['x-registry-delivery-time']); + if (h['content-type'] !== 'application/json' || h['content-encoding'] !== undefined || + h['ce-specversion'] !== '1.0' || h['ce-source'] !== config.expectedSource || + !binding || h['ce-dataschema'] !== schema || !UUID.test(h['ce-id']) || + !DATE.test(h['ce-time']) || !Number.isFinite(Date.parse(h['ce-time'])) || + !DATE.test(h['x-registry-delivery-time']) || !Number.isFinite(delivered) || + Math.abs(Date.now() - delivered) > config.maxDeliverySkewSeconds * 1000 || + !DIGEST.test(h['idempotency-key']) || + !['x-registry-event-generation', 'x-registry-delivery-attempt'].every(name => + /^[1-9]\d*$/.test(h[name]) && Number.isSafeInteger(Number(h[name])))) { + throw new Refusal(400, 'invalid_event'); + } + return binding; +} + +function readBody(request, limit) { + return new Promise((resolve, reject) => { + const chunks = []; + let size = 0; + let done = false; + const finish = (error, body) => { + if (done) return; + done = true; + clearTimeout(timer); + error ? reject(error) : resolve(body); + }; + const timer = setTimeout(() => finish(new Refusal(408, 'request_timeout')), 5000); + request.on('data', chunk => { + if (done) return; + size += chunk.length; + if (size > limit) finish(new Refusal(413, 'body_too_large')); + else chunks.push(chunk); + }); + request.on('end', () => finish(size ? null : new Refusal(400, 'invalid_event'), Buffer.concat(chunks))); + request.on('error', () => finish(new Refusal(400, 'invalid_event'))); + }); +} + +function verifyPayload(body, config, binding) { + let data; + try { data = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(body)); } + catch { throw new Refusal(400, 'invalid_event'); } + const keys = ['entity', 'recordId', 'revision', 'trigger', 'packageRevision', 'values']; + if (!object(data) || Object.keys(data).length !== keys.length || keys.some(key => !Object.hasOwn(data, key)) || + data.entity !== config.expectedEntity || typeof data.recordId !== 'string' || !UUID.test(data.recordId) || + !Number.isSafeInteger(data.revision) || data.revision < 1 || + typeof data.packageRevision !== 'string' || !DIGEST.test(data.packageRevision) || + !['created', 'patched'].includes(data.trigger) || + data.trigger !== binding.trigger || !object(data.values) || + Object.keys(data.values).length !== config.allowedValueFields.length || + config.allowedValueFields.some(field => !Object.hasOwn(data.values, field) || + typeof data.values[field] !== 'string' || Buffer.byteLength(data.values[field]) > 1024)) { + throw new Refusal(400, 'invalid_event'); + } + return data; +} + +function forward(config, envelope) { + return new Promise((resolve, reject) => { + const bytes = Buffer.from(JSON.stringify(envelope)); + const transport = config.openfnUrl.protocol === 'https:' ? https : http; + const request = transport.request(config.openfnUrl, { + method: 'POST', headers: { 'content-type': 'application/json', 'content-length': bytes.length, + 'x-api-key': config.apiKey }, + }, response => { + const chunks = []; + let size = 0; + response.on('data', chunk => { + size += chunk.length; + if (size > 16384) { response.destroy(); reject(new Refusal(502, 'openfn_not_accepted')); } + else chunks.push(chunk); + }); + response.on('error', () => reject(new Refusal(502, 'openfn_unavailable'))); + response.on('end', () => { + try { + const accepted = JSON.parse(Buffer.concat(chunks).toString('utf8')); + if (response.statusCode < 200 || response.statusCode >= 300 || + !object(accepted) || typeof accepted.work_order_id !== 'string' || !UUID.test(accepted.work_order_id) || + Object.hasOwn(accepted, 'error')) throw new Error(); + resolve(); + } catch { reject(new Refusal(502, 'openfn_not_accepted')); } + }); + }); + const timer = setTimeout(() => request.destroy(new Refusal(504, 'openfn_timeout')), config.timeoutMs); + request.on('error', error => reject(error instanceof Refusal ? error : new Refusal(502, 'openfn_unavailable'))); + request.on('close', () => clearTimeout(timer)); + request.end(bytes); + }); +} + +function respond(response, status, code) { + if (!response.destroyed && !response.writableEnded) { + response.writeHead(status, { 'content-type': 'application/json', 'cache-control': 'no-store', + ...(status >= 400 ? { connection: 'close' } : {}) }); + response.end(JSON.stringify({ status: code })); + } +} + +export function createBridge(config) { + const server = http.createServer({ maxHeaderSize: 16384 }, async (request, response) => { + try { + if (request.url === '/healthz' && request.method === 'GET') return respond(response, 200, 'ok'); + if (request.url !== EVENT_PATH || request.method !== 'POST') return respond(response, 404, 'not_found'); + const binding = verifyHeaders(request, config); + const body = await readBody(request, config.maxBodyBytes); + const provided = request.headers['x-registry-signature']; + const expected = webhookSignature(config.hmacKey, request.headers, body); + if (!/^v1=[A-Za-z0-9_-]{43}$/.test(provided) || + !timingSafeEqual(Buffer.from(provided), Buffer.from(expected))) throw new Refusal(401, 'invalid_signature'); + const data = verifyPayload(body, config, binding); + const h = request.headers; + await forward(config, { + event: Object.fromEntries(['specversion', 'id', 'source', 'type', 'time', 'dataschema'].map(key => [key, h[`ce-${key}`]])), + delivery: { generation: Number(h['x-registry-event-generation']), attempt: Number(h['x-registry-delivery-attempt']), + time: h['x-registry-delivery-time'], idempotencyKey: h['idempotency-key'] }, + data, + }); + respond(response, 202, 'accepted'); + } catch (error) { respond(response, error instanceof Refusal ? error.status : 503, error instanceof Refusal ? error.message : 'unavailable'); } + }); + server.requestTimeout = 10000; + server.headersTimeout = 5000; + return server; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + const config = loadConfig(); + const server = createBridge(config); + server.on('error', () => { process.stderr.write('bridge unavailable\n'); process.exitCode = 1; }); + server.listen(config.port, '0.0.0.0'); + for (const signal of ['SIGINT', 'SIGTERM']) process.on(signal, () => server.close()); + } catch { process.stderr.write('invalid bridge configuration\n'); process.exitCode = 1; } +} diff --git a/services/event-bridge/test/bridge.test.js b/services/event-bridge/test/bridge.test.js new file mode 100644 index 0000000..b50c1b2 --- /dev/null +++ b/services/event-bridge/test/bridge.test.js @@ -0,0 +1,189 @@ +import assert from 'node:assert/strict'; +import { createHmac } from 'node:crypto'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import http from 'node:http'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { test } from 'node:test'; +import { createBridge, loadConfig, EVENT_PATH, webhookSignature } from '../src/server.js'; + +const key = Buffer.from('a-synthetic-hmac-key-at-least-32-bytes'); +const source = 'urn:registrystack:registry:agricultural-holdings:instance:pilot'; +const schema = `urn:breg:event-schema:agricultural-holdings:farm:farm-created-v1:sha256:${'a'.repeat(64)}`; +const data = { entity: 'farm', recordId: '11111111-1111-4111-8111-111111111111', revision: 1, + trigger: 'created', packageRevision: `sha256:${'b'.repeat(64)}`, values: { 'local-identifier': 'SYNTHETIC-FARM-001' } }; + +// Independent implementation of the ordered Rust signing contract, deliberately +// not the production verifier helper. Any body or signed-header change is bound. +function sign(headers, body, path = EVENT_PATH, method = 'POST') { + const parts = [Buffer.from('breg-webhook-signature-v1')]; + for (const value of [headers['ce-specversion'], headers['ce-id'], headers['ce-source'], headers['ce-type'], + headers['ce-time'], headers['ce-dataschema'], headers['x-registry-event-generation'], + headers['x-registry-delivery-attempt'], headers['x-registry-delivery-time'], method, path, + headers['content-type'], headers['idempotency-key'], body]) { + const bytes = Buffer.isBuffer(value) ? value : Buffer.from(value); + const length = Buffer.alloc(8); + length.writeBigUInt64BE(BigInt(bytes.length)); + parts.push(length, bytes); + } + return `v1=${createHmac('sha256', key).update(Buffer.concat(parts)).digest('base64url')}`; +} + +function delivery(changes = {}, value = data) { + const body = Buffer.from(JSON.stringify(value)); + const headers = { 'content-type': 'application/json', 'ce-specversion': '1.0', + 'ce-id': '22222222-2222-4222-8222-222222222222', 'ce-source': source, 'ce-type': 'farm-created-v1', + 'ce-time': '2020-01-01T00:00:00Z', 'ce-dataschema': schema, + 'x-registry-event-generation': '1', 'x-registry-delivery-attempt': '1', + 'x-registry-delivery-time': new Date().toISOString(), 'idempotency-key': `sha256:${'c'.repeat(64)}`, + ...changes }; + headers['x-registry-signature'] = sign(headers, body); + return { headers, body }; +} + +async function listen(server) { + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + return `http://127.0.0.1:${server.address().port}`; +} +async function close(server) { + server.closeAllConnections(); + await new Promise(resolve => server.close(resolve)); +} + +async function fixture(t, upstreamHandler, overrides = {}) { + const calls = []; + const upstream = http.createServer(async (request, response) => { + const chunks = []; + for await (const chunk of request) chunks.push(chunk); + calls.push({ headers: request.headers, data: JSON.parse(Buffer.concat(chunks)) }); + if (upstreamHandler) return upstreamHandler(request, response, calls.length); + response.writeHead(201, { 'content-type': 'application/json' }); + response.end(JSON.stringify({ work_order_id: '33333333-3333-4333-8333-333333333333' })); + }); + const upstreamUrl = await listen(upstream); + const config = { hmacKey: key, apiKey: 'synthetic-openfn-key', expectedSource: source, expectedEntity: 'farm', + expectedEvents: { 'farm-created-v1': { schema, trigger: 'created' } }, allowedValueFields: ['local-identifier'], + maxBodyBytes: 65536, maxDeliverySkewSeconds: 300, timeoutMs: 100, openfnUrl: new URL(`${upstreamUrl}/fixed`), ...overrides }; + const bridge = createBridge(config); + const url = await listen(bridge); + t.after(async () => { await close(bridge); await close(upstream); }); + return { url, calls, config }; +} + +test('forwards verified envelope only after real work-order acceptance; old event time is permitted', async t => { + const { url, calls } = await fixture(t); + const request = delivery(); + assert.equal(webhookSignature(key, request.headers, request.body), request.headers['x-registry-signature']); + const response = await fetch(`${url}${EVENT_PATH}`, { method: 'POST', ...request }); + assert.equal(response.status, 202); + assert.deepEqual(await response.json(), { status: 'accepted' }); + assert.equal(calls.length, 1); + assert.equal(calls[0].headers['x-api-key'], 'synthetic-openfn-key'); + assert.equal(calls[0].headers['x-registry-signature'], undefined); + assert.deepEqual(calls[0].data.data, data); + assert.equal(calls[0].data.event.source, source); + assert.equal(calls[0].data.delivery.generation, 1); + assert.equal(JSON.stringify(calls[0]).includes(key.toString()), false); +}); + +test('rejects body and every signed header tampering without upstream effects', async t => { + const { url, calls } = await fixture(t); + for (const field of ['ce-specversion', 'ce-id', 'ce-source', 'ce-type', 'ce-time', 'ce-dataschema', + 'x-registry-event-generation', 'x-registry-delivery-attempt', 'x-registry-delivery-time', + 'content-type', 'idempotency-key', 'x-registry-signature']) { + const request = delivery(); + request.headers[field] += 'x'; + const response = await fetch(`${url}${EVENT_PATH}`, { method: 'POST', ...request }); + assert.ok(response.status >= 400, field); + } + const request = delivery(); + request.body = Buffer.from(request.body.toString().replace('SYNTHETIC-FARM-001', 'SYNTHETIC-FARM-002')); + assert.equal((await fetch(`${url}${EVENT_PATH}`, { method: 'POST', ...request })).status, 401); + assert.equal(calls.length, 0); +}); + +test('rejects correctly signed wrong source/type/schema, old/future delivery and unprojected payload', async t => { + const { url, calls } = await fixture(t); + for (const changes of [ + { 'ce-source': `${source}-other` }, { 'ce-type': 'unexpected' }, { 'ce-dataschema': `${schema}f` }, + { 'x-registry-delivery-time': new Date(Date.now() - 301000).toISOString() }, + { 'x-registry-delivery-time': new Date(Date.now() + 301000).toISOString() }, + ]) assert.equal((await fetch(`${url}${EVENT_PATH}`, { method: 'POST', ...delivery(changes) })).status, 400); + for (const value of [ + { ...data, values: { ...data.values, ownerName: 'SYNTHETIC-PRIVATE-CANARY' } }, + { ...data, trigger: 'patched' }, { ...data, entity: 'other' }, { ...data, revision: 0 }, + { ...data, recordId: [data.recordId] }, { ...data, packageRevision: [data.packageRevision] }, + { ...data, ownerName: 'SYNTHETIC-PRIVATE-CANARY' }, + ]) { + const response = await fetch(`${url}${EVENT_PATH}`, { method: 'POST', ...delivery({}, value) }); + assert.equal(response.status, 400); + assert.equal((await response.text()).includes('SYNTHETIC-PRIVATE-CANARY'), false); + } + assert.equal(calls.length, 0); +}); + +test('fixed method/path, duplicate signed headers and body ceiling reject', async t => { + const { url, calls } = await fixture(t, undefined, { maxBodyBytes: 1024 }); + assert.equal((await fetch(`${url}${EVENT_PATH}?target=other`, { method: 'POST', ...delivery() })).status, 404); + assert.equal((await fetch(`${url}${EVENT_PATH}`, { method: 'PUT', ...delivery() })).status, 404); + assert.equal((await fetch(`${url}${EVENT_PATH}`, { method: 'POST', ...delivery({}, { ...data, extra: 'x'.repeat(1024) }) })).status, 413); + const duplicate = delivery(); + const status = await new Promise((resolve, reject) => { + const headers = Object.entries(duplicate.headers).flat(); + headers.push('CE-ID', duplicate.headers['ce-id']); + const request = http.request(`${url}${EVENT_PATH}`, { method: 'POST', headers }, response => { + response.resume(); response.on('end', () => resolve(response.statusCode)); + }); + request.on('error', reject); request.end(duplicate.body); + }); + assert.equal(status, 400); + assert.equal(calls.length, 0); +}); + +test('upstream failure is retryable and bridge performs no hidden retries', async t => { + const { url, calls } = await fixture(t, (_request, response, attempt) => { + response.writeHead(attempt === 1 ? 503 : 201); + response.end(attempt === 1 ? 'unavailable' : JSON.stringify({ work_order_id: '33333333-3333-4333-8333-333333333333' })); + }); + assert.equal((await fetch(`${url}${EVENT_PATH}`, { method: 'POST', ...delivery() })).status, 502); + assert.equal(calls.length, 1); + assert.equal((await fetch(`${url}${EVENT_PATH}`, { method: 'POST', ...delivery({ 'x-registry-delivery-attempt': '2' }) })).status, 202); + assert.equal(calls.length, 2); +}); + +test('redirect, generic 2xx, incomplete acceptance and oversized response are not acknowledgements', async t => { + for (const [status, body, headers] of [[302, '{}', { location: '/other' }], [200, '{}', {}], + [201, '{"work_order_id":"x"}', {}], + [200, JSON.stringify({work_order_id: '33333333-3333-4333-8333-333333333333', error: 'too_many_runs'}), {}], [200, 'x'.repeat(17000), {}]]) { + await t.test(String(status) + body.length, async t => { + const { url, calls } = await fixture(t, (_request, response) => { response.writeHead(status, headers); response.end(body); }); + assert.equal((await fetch(`${url}${EVENT_PATH}`, { method: 'POST', ...delivery() })).status, 502); + assert.equal(calls.length, 1); + }); + } +}); + +test('timeout and lost upstream acceptance response remain retryable', async t => { + const { url, calls } = await fixture(t, (request, _response, count) => { if (count === 2) request.socket.destroy(); }); + assert.equal((await fetch(`${url}${EVENT_PATH}`, { method: 'POST', ...delivery() })).status, 504); + assert.equal(calls.length, 1); + assert.equal((await fetch(`${url}${EVENT_PATH}`, { method: 'POST', ...delivery({ 'x-registry-delivery-attempt': '2' }) })).status, 502); + assert.equal(calls.length, 2); +}); + +test('configuration uses exact secret bytes, explicit HTTP trust and safe errors', async t => { + const dir = await mkdtemp(join(tmpdir(), 'bridge-config-')); + t.after(() => rm(dir, { recursive: true, force: true })); + await Promise.all([ + writeFile(join(dir, 'hmac'), Buffer.concat([key, Buffer.from('\n')])), + writeFile(join(dir, 'api'), 'SYNTHETIC-API-KEY'), + writeFile(join(dir, 'events'), JSON.stringify({ 'farm-created-v1': { schema, trigger: 'created' } })), + writeFile(join(dir, 'fields'), '["local-identifier"]'), + ]); + const env = { BREG_HMAC_KEY_FILE: join(dir, 'hmac'), OPENFN_API_KEY_FILE: join(dir, 'api'), + BREG_EXPECTED_EVENTS_FILE: join(dir, 'events'), BREG_ALLOWED_VALUE_FIELDS_FILE: join(dir, 'fields'), + BREG_EXPECTED_SOURCE: source, BREG_EXPECTED_ENTITY: 'farm', OPENFN_WEBHOOK_URL: 'http://openfn:4000/i/pilot' }; + assert.throws(() => loadConfig(env), /^Error: invalid bridge configuration$/); + assert.equal(loadConfig({ ...env, ALLOW_HTTP: 'true' }).hmacKey.at(-1), 10); + assert.throws(() => loadConfig({ ...env, OPENFN_WEBHOOK_URL: 'https://secret:canary@example.org/' }), /^Error: invalid bridge configuration$/); +});