Add self-hosted Evidence and BREG OpenFn pilot

.dockerignore

@@ -0,0 +1,11 @@
+.git
+node_modules
+**/node_modules
+**/__pycache__
+pilot/agriculture/.runtime
+deployment/.env
+deployment/.runtime
+deployment/.build
+coverage
+*.tgz
+npm-debug.log*

.github/workflows/ci.yml

@@ -1,19 +1,66 @@
-name: CI
+name: Registry OpenFn pilot
 
 on:
   pull_request:
   push:
-    branches:
-      - main
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: registry-openfn-pilot-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
-  test:
-    runs-on: ubuntu-latest
+  adaptor-and-service-checks:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: 22
+          node-version: 24.19.0
           cache: npm
-      - run: npm ci --ignore-scripts --no-audit --no-fund
-      - run: npm run check
+      - name: Install the committed npm dependency graph
+        run: npm ci --ignore-scripts --no-audit --no-fund
+      - name: Check Evidence, BREG, event services and compiled pilot jobs
+        run: npm run check
+      - name: Check adaptor package contents
+        run: npm run pack:dry-run
+
+  linux-pilot-acceptance:
+    needs: adaptor-and-service-checks
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build pinned images, execute the native worker gate and set up the pilot
+        # Setup owns the build and suppresses raw upstream subprocess output.
+        run: ./deployment/pilot.sh setup
+      - name: Check private mount ownership
+        run: ./deployment/check-permissions.sh
+      - name: Run registration, governed correction and verified destination acceptance
+        run: python3 pilot/smoke.py --timeout 300
+      - name: Verify retained records and exact replay after service restart
+        run: python3 pilot/smoke.py --restart-recovery --timeout 300
+      - name: Verify delivery retry, failed-step replay and queued-work recovery
+        run: python3 pilot/recovery.py --scenario all --timeout 300
+      - name: Report service status without logs or credentials
+        if: failure()
+        shell: bash
+        run: |
+          if [ -f pilot/agriculture/.runtime/prepared.json ]; then
+            ./deployment/pilot.sh status
+          fi
+      - name: Stop application services and retain isolated test data until runner disposal
+        if: always()
+        shell: bash
+        run: |
+          if [ -f pilot/agriculture/.runtime/prepared.json ]; then
+            if ! ./deployment/pilot.sh stop >/dev/null 2>&1; then
+              echo 'Pilot stop did not complete. The isolated CI runner will be disposed.'
+            fi
+          fi
+          # Never upload runtime files, private keys, dataclips or raw service logs.

.gitignore

@@ -3,3 +3,8 @@ node_modules/
 npm-debug.log*
 .DS_Store
 coverage/
+pilot/agriculture/.runtime/
+deployment/.env
+deployment/.build/
+deployment/.runtime/
+__pycache__/

deployment/.gitignore

@@ -0,0 +1,3 @@
+.env
+.runtime/
+__pycache__/

deployment/Lightning.Dockerfile

@@ -0,0 +1,8 @@
+# syntax=docker/dockerfile:1
+ARG LIGHTNING_IMAGE=openfn/lightning:v2.18.2@sha256:a2327173ece7b5ac4904744acf63f928dd2cbe496788d31266060e85bd359203
+ARG ADAPTORS_IMAGE=registry-openfn-worker:pilot
+FROM ${ADAPTORS_IMAGE} AS adaptors
+FROM ${LIGHTNING_IMAGE}
+COPY --from=adaptors --chown=lightning:root /opt/registry-adaptors /opt/registry-adaptors
+ENV OPENFN_ADAPTORS_REPO=/opt/registry-adaptors
+ENV LOCAL_ADAPTORS=true

deployment/Lightning.Dockerfile.dockerignore

@@ -0,0 +1 @@
+**

deployment/Postgres.Dockerfile

@@ -0,0 +1,4 @@
+FROM postgres:17.11@sha256:67f41722b7a8cbdb868a44a4995c846eddfdc2973bccb291ce937dce88ad5675
+COPY --chmod=755 deployment/postgres-tls-entrypoint.sh /usr/local/bin/pilot-postgres-entrypoint
+ENTRYPOINT ["/usr/local/bin/pilot-postgres-entrypoint"]
+CMD ["postgres", "-c", "ssl=on", "-c", "ssl_cert_file=/var/lib/postgresql/tls/server.crt", "-c", "ssl_key_file=/var/lib/postgresql/tls/server.key"]

deployment/Postgres.Dockerfile.dockerignore

@@ -0,0 +1,3 @@
+**
+!deployment
+!deployment/postgres-tls-entrypoint.sh

deployment/README.md

@@ -0,0 +1,98 @@
+# Pilot container packaging
+
+This directory packages upstream Lightning 2.18.2 and websocket worker 1.29.0
+with the local Evidence and BREG adaptors. Upstream and
+Registry Stack image identities are recorded in `images.lock.json`.
+
+Run `./deployment/build-images.sh` from the repository after its package lock is
+current. It builds the worker first and compiles the exact four generated
+Lightning job bodies with upstream adaptor export preloading and automatic
+imports. The BREG and Evidence jobs must reach their own input validation, and
+the destination job executes against a controlled loopback responder. The check
+also proves credentials remain available between operations and disappear at
+the normal job boundary.
+The released engine's real child-process handshake is checked separately under
+the image's configured UID and GID, with no network access.
+It then builds Lightning, the event services, PostgreSQL TLS setup, and the
+checksum-verified Registry Stack 0.27.0 bootstrap tools.
+The PostgreSQL check initializes an isolated database from synthetic files owned
+by UID1001 with mode 0600, proving that its runtime user can read the staged SQL
+without relaxing the operator's file permissions. It mounts no pilot data.
+Bootstrap packages come from the signed Debian snapshot dated August 24, 2026,
+matching the pinned PostgreSQL base image.
+
+The worker release is Alpine. The worker Dockerfile copies its compiled
+application unchanged into pinned Node 24.19.0 Bookworm, installs the upstream
+frozen dependency lock for glibc, and installs the adaptor repository's locked
+production dependencies. It does not rebuild or patch OpenFn source. The local
+adaptor root contains `registry-evidence`, `registry-breg`, `common`, and `http`.
+No dependency installation is performed when the worker starts.
+
+The released Lightning and worker images provide **linux/amd64 only**. This
+pilot therefore pins that platform, including on Apple Silicon under Docker
+emulation. The arm64 adaptor layer independently passed both Evidence and BREG native
+constructor checks on Node 24.19.0/glibc 2.36. Reproduce with
+`docker build --platform linux/arm64 --target adaptors -f deployment/Worker.Dockerfile .`.
+An arm64 native-worker claim requires an upstream arm64 image and a
+separate real-worker acceptance run. The Node and Registry Stack clients alone
+being available for arm64 does not establish that claim.
+
+`compose.yaml` belongs to the synthetic agriculture pilot. Generate its ignored
+runtime configuration using the pilot's setup workflow before using Compose.
+It contains persistent independent Lightning and BREG databases. Mint owns a
+shared network namespace for BREG, Evidence, worker, bridge and bootstrap tools,
+so Registry Stack's explicit development HTTP policy uses loopback. Only the
+Lightning UI (4000), BREG event bridge (4001) and BREG reviewer API (4002) are
+published, all on host loopback.
+
+Configuration mounts are scoped to each service. In particular the worker only
+receives its Evidence client profile, and Lightning receives its own bootstrap
+secrets. The bootstrap tools alone receive all generated Registry Stack inputs.
+The wrapper copies the database's private key and initial SQL with postgres
+ownership and mode 0600 before invoking the unchanged upstream entrypoint. The
+initial SQL applies only when PostgreSQL initializes a new named volume.
+
+Follow the pilot runbook for migrations, first operator setup, provisioning,
+and acceptance. Build checks establish packaging and native execution; they do
+not by themselves establish the full workflow or event-delivery journey.
+
+Use `./deployment/pilot.sh setup` for the complete bootstrap, then `start`,
+`stop`, or `status`. Setup preserves an existing completed runtime and verifies
+its retained package; it does not clear a failed preparation or reset records.
+A failed step stops startup, and raw runtime diagnostics are suppressed because
+upstream errors can contain short-lived worker-token URLs. Database migrations
+use the upstream release API before starting Lightning. Workflows are provisioned
+disabled, webhook authentication is attached, then authentication is checked
+before enabling triggers. Readiness waits are bounded.
+
+`./deployment/compose.sh` is the supported Compose entrypoint. It creates ignored
+`deployment/.env` containing the current operator UID and GID and uses that same
+identity for OpenFn, Registry Stack, bootstrap and event-service containers.
+The worker image also bakes a matching passwd/group entry when needed because
+upstream starts its engine children with an empty environment. Rebuild images
+with `build-images.sh` when transferring the pilot to a different operator.
+Generated private files remain owned by that operator with restrictive modes.
+The separate PostgreSQL entrypoint still drops to the image's postgres user.
+`volume-init` adjusts ownership only inside the pilot's named audit, destination
+and worker-cache volumes. It preserves metadata when ownership and permissions
+already match, so resumed starts do not invalidate active audit writers.
+No other host paths or volumes are touched.
+
+Run `./deployment/check-permissions.sh` after preparing or transferring a pilot.
+It checks each service's private bind mount under the selected UID without
+printing contents. Moving a pilot to another operator requires an explicit
+ownership transfer of retained private files and updating `deployment/.env`;
+the wrapper refuses to silently use another operator's settings. Do not make
+private keys world-readable. Local image checks use a non-default UID/GID on
+macOS/OrbStack. The focused GitHub Actions gate runs fresh setup, the complete
+workflow journey, retained restart and failure recovery on Ubuntu 24.04.
+
+External Sentry reporting is explicitly disabled for worker and Lightning, and
+Lightning usage tracking is disabled. The worker uses its dedicated writable
+cache rather than writing to the packaged application.
+
+Reset is deliberately separate and destructive. Only
+`./deployment/pilot.sh reset --confirm-delete-synthetic-data` removes the fixed
+`registry-openfn-pilot` project volumes and its marked synthetic `.runtime`
+directory. No reset is performed by setup, start or stop. Run the normal pilot
+smoke journey after setup to establish full workflow behavior.

deployment/Services.Dockerfile

@@ -0,0 +1,7 @@
+# syntax=docker/dockerfile:1
+FROM node:24.19.0-bookworm@sha256:4196d66a565c6f195728d9952f161f4adfe2ad753052a08b7ec7f1c5a6bda42b
+WORKDIR /app
+COPY services /app/services
+RUN mkdir /data && chown node:node /data
+USER node
+CMD ["node", "services/event-bridge/src/server.js"]

deployment/Services.Dockerfile.dockerignore

@@ -0,0 +1,5 @@
+**
+!services
+!services/**
+services/**/node_modules
+services/**/.env

deployment/Tools.Dockerfile

@@ -0,0 +1,25 @@
+# Debian 13 matches the released Linux tools' runtime. PostgreSQL client is
+# already supplied by the same pinned image used for the pilot database.
+FROM postgres:17.11@sha256:67f41722b7a8cbdb868a44a4995c846eddfdc2973bccb291ce937dce88ad5675
+# Match the immutable Debian snapshot recorded by the pinned base image.
+# Retain signature verification; historical repository metadata has expired.
+RUN sed -i \
+      -e 's|http://deb.debian.org/debian-security|http://snapshot.debian.org/archive/debian-security/20260824T000000Z|' \
+      -e 's|http://deb.debian.org/debian|http://snapshot.debian.org/archive/debian/20260824T000000Z|' \
+      /etc/apt/sources.list.d/debian.sources && \
+    rm /etc/apt/sources.list.d/pgdg.list && \
+    apt-get -o Acquire::Check-Valid-Until=false -o APT::Update::Error-Mode=any update && \
+    apt-get install -y --no-install-recommends python3 python3-yaml curl ca-certificates openssl && \
+    rm -rf /var/lib/apt/lists/*
+COPY deployment/tools.sha256 /tmp/tools.sha256
+RUN set -eu; cd /tmp; \
+    for tool in breg bregctl evidence evidencectl mint; do \
+      curl --fail --location --retry 3 --output "$tool-v0.27.0-linux-amd64" "https://github.com/registrystack/registry-stack/releases/download/v0.27.0/$tool-v0.27.0-linux-amd64"; \
+    done; \
+    sha256sum --check tools.sha256; \
+    for tool in breg bregctl evidence evidencectl mint; do \
+      install -m 0755 "$tool-v0.27.0-linux-amd64" "/usr/local/bin/$tool"; \
+      rm "$tool-v0.27.0-linux-amd64"; \
+    done
+ENTRYPOINT []
+CMD ["python3", "--version"]

deployment/Tools.Dockerfile.dockerignore

@@ -0,0 +1,3 @@
+**
+!deployment
+!deployment/tools.sha256

deployment/Worker.Dockerfile

@@ -0,0 +1,49 @@
+# syntax=docker/dockerfile:1
+ARG WORKER_IMAGE=openfn/ws-worker:v1.29.0@sha256:827f4897d120c322d977da7bc2810174332eb5ba4902fdb7279178812d787c18
+ARG NODE_IMAGE=node:24.19.0-bookworm@sha256:4196d66a565c6f195728d9952f161f4adfe2ad753052a08b7ec7f1c5a6bda42b
+FROM ${WORKER_IMAGE} AS upstream
+FROM ${NODE_IMAGE} AS worker-runtime
+# Upstream publishes Alpine only. Keep its compiled application unchanged and
+# re-install the exact locked dependencies for glibc. Never rebuild its source.
+COPY --from=upstream /app /app
+WORKDIR /app
+RUN corepack enable && CI=true pnpm install --frozen-lockfile --force --ignore-scripts
+WORKDIR /app/packages/ws-worker
+USER node
+CMD ["node", "./dist/start.js"]
+
+FROM ${NODE_IMAGE} AS adaptors
+WORKDIR /opt/registry-adaptors
+COPY package.json package-lock.json ./
+COPY packages/registry-evidence/package.json packages/registry-evidence/package.json
+COPY packages/registry-breg/package.json packages/registry-breg/package.json
+RUN npm ci --omit=dev --ignore-scripts --no-audit --no-fund
+COPY deployment/check-native.cjs /opt/registry-adaptors/check-native.cjs
+RUN node /opt/registry-adaptors/check-native.cjs
+COPY packages/registry-evidence/src packages/registry-evidence/src
+COPY packages/registry-evidence/configuration-schema.json packages/registry-evidence/configuration-schema.json
+COPY packages/registry-breg/src packages/registry-breg/src
+COPY packages/registry-breg/configuration-schema.json packages/registry-breg/configuration-schema.json
+# Local loading expects packages/<adaptor>, including common and HTTP.
+RUN ln -s ../node_modules/@openfn/language-common packages/common && \
+    ln -s ../node_modules/@openfn/language-http packages/http
+
+FROM worker-runtime AS worker
+ARG PILOT_UID=1000
+ARG PILOT_GID=1000
+USER root
+COPY --from=adaptors --chown=node:node /opt/registry-adaptors /opt/registry-adaptors
+COPY deployment/check-worker.mjs /opt/registry-adaptors/check-worker.mjs
+COPY deployment/check-engine.mjs /opt/registry-adaptors/check-engine.mjs
+COPY pilot/lightning/project.mjs /opt/registry-adaptors/pilot-project.mjs
+ENV OPENFN_ADAPTORS_REPO=/opt/registry-adaptors
+ENV LOCAL_ADAPTORS=true
+RUN node --experimental-vm-modules /opt/registry-adaptors/check-worker.mjs
+# The upstream engine starts children with an empty environment. Their Node
+# runtime needs a passwd entry for the mapped operator UID during startup.
+RUN if ! getent group "$PILOT_GID" >/dev/null; then groupadd --gid "$PILOT_GID" pilot; fi && \
+    if ! getent passwd "$PILOT_UID" >/dev/null; then \
+      useradd --uid "$PILOT_UID" --gid "$PILOT_GID" --home-dir /var/cache/openfn --no-create-home --shell /usr/sbin/nologin pilot; \
+    fi && \
+    mkdir -p /var/cache/openfn && chown "$PILOT_UID:$PILOT_GID" /var/cache/openfn && chmod 0700 /var/cache/openfn
+USER ${PILOT_UID}:${PILOT_GID}

deployment/Worker.Dockerfile.dockerignore

@@ -0,0 +1,21 @@
+**
+!package.json
+!package-lock.json
+!packages
+!packages/registry-evidence
+!packages/registry-evidence/package.json
+!packages/registry-evidence/configuration-schema.json
+!packages/registry-evidence/src
+!packages/registry-evidence/src/**
+!packages/registry-breg
+!packages/registry-breg/package.json
+!packages/registry-breg/configuration-schema.json
+!packages/registry-breg/src
+!packages/registry-breg/src/**
+!deployment
+!deployment/check-worker.mjs
+!deployment/check-engine.mjs
+!deployment/check-native.cjs
+!pilot
+!pilot/lightning
+!pilot/lightning/project.mjs

deployment/build-images.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+set -eu
+cd "$(dirname "$0")/.."
+docker build --platform linux/amd64 --target worker --build-arg "PILOT_UID=$(id -u)" --build-arg "PILOT_GID=$(id -g)" -f deployment/Worker.Dockerfile -t registry-openfn-worker:pilot .
+docker run --rm --platform linux/amd64 --entrypoint node registry-openfn-worker:pilot --experimental-vm-modules /opt/registry-adaptors/check-worker.mjs
+docker run --rm --platform linux/amd64 --network none --entrypoint node registry-openfn-worker:pilot /opt/registry-adaptors/check-engine.mjs
+docker build --platform linux/amd64 -f deployment/Lightning.Dockerfile -t registry-openfn-lightning:pilot .
+docker build --platform linux/amd64 -f deployment/Services.Dockerfile -t registry-openfn-services:pilot .
+docker build --platform linux/amd64 -f deployment/Postgres.Dockerfile -t registry-openfn-postgres:pilot .
+sh deployment/check-postgres.sh
+docker build --platform linux/amd64 -f deployment/Tools.Dockerfile -t registry-openfn-tools:pilot .

deployment/check-engine.mjs

@@ -0,0 +1,24 @@
+// Exercise the released engine's child-process handshake, including its empty
+// child environment, under the image's configured runtime identity.
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import createEngine from '/app/packages/engine-multi/dist/index.js';
+
+const repoDir = await mkdtemp(join(tmpdir(), 'pilot-engine-gate-'));
+const timeout = setTimeout(() => {
+  console.error('Released worker engine did not initialize within 20 seconds.');
+  process.exit(1);
+}, 20_000);
+let engine;
+try {
+  engine = await createEngine({
+    repoDir, memoryLimitMb: 500, maxWorkers: 2,
+    workerValidationTimeout: 5000, workerValidationRetries: 1,
+  });
+  console.log(JSON.stringify({ verified: 'released-worker-engine-startup', uid: process.getuid(), gid: process.getgid() }));
+} finally {
+  if (engine) await engine.destroy();
+  clearTimeout(timeout);
+  await rm(repoDir, { recursive: true, force: true });
+}

deployment/check-native.cjs

@@ -0,0 +1,11 @@
+const assert = require('node:assert/strict');
+const { generateKeyPairSync, createHash } = require('node:crypto');
+const { createRequire } = require('node:module');
+const { evidence, breg } = createRequire('/opt/registry-adaptors/package.json')('@registrystack/client');
+const { publicKey } = generateKeyPairSync('ec', { namedCurve: 'prime256v1' });
+const jwk = publicKey.export({ format: 'jwk' });
+const kid = createHash('sha256').update(JSON.stringify({ crv: jwk.crv, kty: jwk.kty, x: jwk.x, y: jwk.y })).digest('base64url');
+new evidence.EvidenceClient({ baseUrl: 'http://127.0.0.1:8080', trustedJwks: { keys: [{ ...jwk, kid, alg: 'ES256' }] }, revokedKeyIds: [], token: { static: 'synthetic-test-value' } });
+new breg.BaseRegistryClient({ baseUrl: 'http://127.0.0.1:8090' });
+assert.ok(process.report.getReport().header.glibcVersionRuntime);
+console.log(JSON.stringify({ verified: 'native-evidence-and-breg-constructors', architecture: process.arch, node: process.version, glibc: process.report.getReport().header.glibcVersionRuntime }));