From 523014c2b089bbfa032a94f72b18261b209beddd Mon Sep 17 00:00:00 2001 From: Val Redchenko Date: Mon, 1 Jun 2026 21:13:36 +0100 Subject: [PATCH] feat(ci): automate OpenAPI spec sync from backend, add ADR 0020 Add ADR 0020 (OpenAPI spec pipeline and version compatibility) and the receiver side of the pipeline: a `Sync OpenAPI spec from backend` workflow that, on a repository_dispatch from smartem-decisions (or manual/scheduled fallback), refreshes the committed swagger caches and rebuilds Pages by calling deploy-webui as a reusable workflow (workflow_call with a ref input, so it deploys the freshly-synced commit). Align docs and Claude Code guidance with the new flow: smartem-decisions is the canonical spec publisher; the devtools and frontend specs are downstream caches; the frontend's version check becomes semantic and observe-only. --- .github/workflows/deploy-webui.yml | 9 +++ .github/workflows/sync-openapi-spec.yml | 70 +++++++++++++++++++ claude-code/ARCHITECTURE.md | 8 ++- claude-code/CLAUDE.md | 2 +- .../smartem-decisions/REPO-GUIDELINES.md | 2 +- .../smartem-frontend/REPO-GUIDELINES.md | 22 +++--- docs/backend/api-documentation.md | 2 +- docs/decision-records/decisions.md | 1 + ...spec-pipeline-and-version-compatibility.md | 61 ++++++++++++++++ docs/development/tools.md | 12 ++-- 10 files changed, 171 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/sync-openapi-spec.yml create mode 100644 docs/decision-records/decisions/0020-openapi-spec-pipeline-and-version-compatibility.md diff --git a/.github/workflows/deploy-webui.yml b/.github/workflows/deploy-webui.yml index 1fd3226..87c1a07 100644 --- a/.github/workflows/deploy-webui.yml +++ b/.github/workflows/deploy-webui.yml @@ -1,6 +1,13 @@ name: Deploy WebUI to GitHub Pages on: + workflow_call: + inputs: + ref: + description: 'Git ref to build (defaults to the triggering ref)' + required: false + type: string + default: '' workflow_dispatch: push: branches: [main] @@ -23,6 +30,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref }} - name: Setup Node.js uses: actions/setup-node@v6 diff --git a/.github/workflows/sync-openapi-spec.yml b/.github/workflows/sync-openapi-spec.yml new file mode 100644 index 0000000..46a0a38 --- /dev/null +++ b/.github/workflows/sync-openapi-spec.yml @@ -0,0 +1,70 @@ +name: Sync OpenAPI spec from backend + +# Downstream cache of the canonical SmartEM OpenAPI spec (ADR 0020). smartem-decisions +# fires a repository_dispatch (openapi-spec-updated) when it publishes a changed spec; +# we pull it, refresh both committed swagger.json copies, and rebuild GitHub Pages. +# workflow_dispatch (manual) and a daily schedule act as fallbacks if a dispatch is missed. + +on: + repository_dispatch: + types: [openapi-spec-updated] + workflow_dispatch: + schedule: + - cron: '17 4 * * *' + +permissions: + contents: write + pages: write + id-token: write + +concurrency: + group: sync-openapi-spec + cancel-in-progress: false + +env: + CANONICAL_SPEC_URL: https://raw.githubusercontent.com/DiamondLightSource/smartem-decisions/main/docs/api/openapi.json + +jobs: + sync: + runs-on: ubuntu-latest + outputs: + changed: ${{ steps.write.outputs.changed }} + steps: + - uses: actions/checkout@v6 + + - name: Download canonical spec + run: | + curl -fsSL "$CANONICAL_SPEC_URL" -o /tmp/openapi.json + python3 -c "import json; json.load(open('/tmp/openapi.json'))" # validate JSON + + - name: Refresh committed copies + detect change + id: write + run: | + changed=false + for dest in docs/api/smartem/swagger.json webui/public/api/smartem/swagger.json; do + mkdir -p "$(dirname "$dest")" + if ! cmp -s /tmp/openapi.json "$dest"; then changed=true; fi + cp /tmp/openapi.json "$dest" + done + echo "changed=$changed" >> "$GITHUB_OUTPUT" + echo "Spec changed: $changed" + + - name: Commit refreshed copies + if: steps.write.outputs.changed == 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add docs/api/smartem/swagger.json webui/public/api/smartem/swagger.json + git commit -m "chore(api): sync OpenAPI spec from smartem-decisions [skip ci]" + git push origin HEAD:main + + deploy: + needs: sync + if: needs.sync.outputs.changed == 'true' + permissions: + contents: read + pages: write + id-token: write + uses: ./.github/workflows/deploy-webui.yml + with: + ref: main diff --git a/claude-code/ARCHITECTURE.md b/claude-code/ARCHITECTURE.md index d7e5d6f..f1bf51c 100644 --- a/claude-code/ARCHITECTURE.md +++ b/claude-code/ARCHITECTURE.md @@ -90,6 +90,10 @@ smartem-backend API serves multiple consumers with different needs: | Frontend client | TBD | smartem-frontend | May need SSE for live updates | | Deposition client | No | fandanGO-cryoem-dls | REST only | +### OpenAPI spec flow (ADR 0020) + +smartem-decisions is the **canonical OpenAPI spec publisher**. On push to main, when the API surface changes, it regenerates and commits the spec at `docs/api/openapi.json`, then fires a `repository_dispatch` to smartem-devtools. The receiver there refreshes its committed swagger copies under `docs/api/smartem/` and `webui/public/api/smartem/` and rebuilds GitHub Pages. smartem-frontend's `packages/api/src/openapi.json` is a downstream cache refreshed from the canonical backend spec (`npm run api:fetch`); both downstream copies are caches, never hand-maintained. The backend exposes a `/version` endpoint; the frontend runs an observe-only, semantic version check against it at boot. The agent ships from the same repo/tag as the backend, so it is version-locked by construction. + ## Mocking Requirements for E2E Testing | External Dependency | Mock Strategy | Status | @@ -150,10 +154,10 @@ export ARIA_GQL_LOCAL=http://localhost:9002/graphql | Affected | Action Required | |----------|-----------------| -| smartem-frontend | Regenerate OpenAPI client | +| smartem-frontend | Refresh cached spec from the canonical backend (`npm run api:update`) + regenerate client | | smartem-agent | Update api_client imports | | fandanGO-cryoem-dls | Update SmartEMAPIClient | -| Docs | Regenerate OpenAPI spec | +| Docs (smartem-devtools) | Auto-refreshed — backend `repository_dispatch` updates the swagger caches and redeploys Pages (no manual step; ADR 0020) | | Containers | Rebuild images | ### When smartem-backend MQ schema changes diff --git a/claude-code/CLAUDE.md b/claude-code/CLAUDE.md index fc66a7a..c88202d 100644 --- a/claude-code/CLAUDE.md +++ b/claude-code/CLAUDE.md @@ -82,7 +82,7 @@ The primary repo of the workspace. Originally the only repo, now the hub of a mu Pure SPA that talks to smartem-decisions backend API. Hosted in proximity to backend. -- Auto-generated API client from backend OpenAPI spec +- API client auto-generated from a cached copy of the canonical backend OpenAPI spec (ADR 0020) - Models route for viewing ML prediction models **Tech**: React 19, TanStack Router, Material-UI, Node.js 22+, Biome, Lefthook diff --git a/claude-code/smartem-decisions/REPO-GUIDELINES.md b/claude-code/smartem-decisions/REPO-GUIDELINES.md index e0e6c3c..02ebe25 100644 --- a/claude-code/smartem-decisions/REPO-GUIDELINES.md +++ b/claude-code/smartem-decisions/REPO-GUIDELINES.md @@ -92,7 +92,7 @@ uv run alembic revision --autogenerate -m "Description" ## Documentation - **Markdown**: Documentation in `docs/` synced to smartem-devtools webui as MDX -- **API documentation**: Swagger/OpenAPI specs auto-generated +- **API documentation**: this repo is the canonical OpenAPI spec publisher (ADR 0020). On push to main, when the API surface changes, CI regenerates and commits the spec at `docs/api/openapi.json` and fires a `repository_dispatch` to smartem-devtools, which refreshes its downstream swagger caches and redeploys Pages. The backend also exposes a `/version` endpoint consumed by the frontend's observe-only version check. - **Live development**: Run `npm run dev` in smartem-devtools/webui for hot-reload ## Available Skills diff --git a/claude-code/smartem-frontend/REPO-GUIDELINES.md b/claude-code/smartem-frontend/REPO-GUIDELINES.md index a6a4469..2fb3cd7 100644 --- a/claude-code/smartem-frontend/REPO-GUIDELINES.md +++ b/claude-code/smartem-frontend/REPO-GUIDELINES.md @@ -40,7 +40,7 @@ smartem-frontend/ │ │ │ ├── mutator.ts # Axios configuration │ │ │ ├── stubs.ts # Development stubs │ │ │ └── index.ts # Barrel export -│ │ ├── openapi.json # OpenAPI spec (version controlled) +│ │ ├── openapi.json # Cached copy of the canonical backend spec (committed; refreshed via api:fetch) │ │ └── orval.config.ts │ └── ui/ # Shared UI component library (@smartem/ui) │ └── src/ @@ -62,9 +62,11 @@ npm run dev:smartem # New app dev server npm run dev:smartem:mock # New app with mock API data # API Client -npm run api:update # Fetch OpenAPI spec + regenerate client +npm run api:update # Refresh cached spec from the canonical backend, then regenerate client npm run api:fetch:local # Fetch from local backend (localhost:8000) -npm run api:generate # Regenerate client from current spec +npm run api:generate # Regenerate client from the committed spec cache +# api:fetch pulls the canonical spec published by smartem-decisions (the backend), +# NOT the devtools GitHub Pages copy. See ADR 0020. # Code Quality npm run check # Biome lint + format (recommended) @@ -80,7 +82,7 @@ npm run build:smartem # Build new app ## API Client Workflow -The frontend uses Orval to generate a type-safe API client from the backend OpenAPI spec. The client lives in `packages/api/` and is shared across both apps as `@smartem/api`. +The frontend uses Orval to generate a type-safe API client from the backend OpenAPI spec. The client lives in `packages/api/` and is shared across both apps as `@smartem/api`. The committed `openapi.json` is a downstream cache of the canonical spec published by smartem-decisions; `npm run api:fetch` refreshes it from the backend (ADR 0020). ### When Backend API Changes @@ -109,12 +111,14 @@ function MyComponent() { } ``` -### Version Mismatch Warning +### API Version Check (observe-only) -If console shows API version mismatch, regenerate the client: -```bash -npm run api:update -``` +On boot the app compares the backend API version its client was built against to +the live backend `/version` endpoint, using a semantic comparison (release portion +only; the `dev`/`+sha` suffix is ignored). A difference is logged as an advisory and, +in development, shown as a non-blocking banner — it is **observed, not enforced**, so +rolling deploys where the two momentarily differ never break the app. If you want the +client rebuilt against the current backend contract, run `npm run api:update`. See ADR 0020. ## Code Quality Tools diff --git a/docs/backend/api-documentation.md b/docs/backend/api-documentation.md index e31392f..c180d3b 100644 --- a/docs/backend/api-documentation.md +++ b/docs/backend/api-documentation.md @@ -83,7 +83,7 @@ You can download the raw OpenAPI specifications: - [Athena API Spec](../api/athena/swagger.json) - Official external specification - [Athena Source Spec](../athena-decision-service-api-spec.json) - Original specification file -- [SmartEM API Spec](../api/smartem/swagger.json) - Generated from our implementation +- [SmartEM API Spec](../api/smartem/swagger.json) - Published by the smartem-decisions backend and cached here automatically (ADR 0020) ### Using Specifications diff --git a/docs/decision-records/decisions.md b/docs/decision-records/decisions.md index 767cc58..16f1955 100644 --- a/docs/decision-records/decisions.md +++ b/docs/decision-records/decisions.md @@ -22,5 +22,6 @@ Architectural decisions are made throughout a project's lifetime. As a way of ke - [ADR-0016: Facility Connector Fork Synchronization](/docs/explanations/decisions/0016-facility-connector-fork-sync) (Proposed) - [ADR-0017: SmartEM Frontend Monorepo Restructure](/docs/explanations/decisions/0017-smartem-frontend-monorepo-restructure) - [ADR-0019: SmartEM Frontend Release and Deployment Pipeline](/docs/explanations/decisions/0019-smartem-frontend-release-pipeline) +- [ADR-0020: SmartEM OpenAPI Specification Pipeline and Version Compatibility](/docs/explanations/decisions/0020-openapi-spec-pipeline-and-version-compatibility) For more information on ADRs see this [blog by Michael Nygard](http://thinkrelevance.com/blog/2011/11/15/documenting-architecture-decisions). diff --git a/docs/decision-records/decisions/0020-openapi-spec-pipeline-and-version-compatibility.md b/docs/decision-records/decisions/0020-openapi-spec-pipeline-and-version-compatibility.md new file mode 100644 index 0000000..6911ba3 --- /dev/null +++ b/docs/decision-records/decisions/0020-openapi-spec-pipeline-and-version-compatibility.md @@ -0,0 +1,61 @@ +# 20. SmartEM OpenAPI Specification Pipeline and Version Compatibility + +Date: 2026-06-01 + +## Status + +Accepted + +## Context + +The SmartEM OpenAPI specification exists as **three independently committed copies** with no automation keeping them in step: + +1. `smartem-decisions` — the FastAPI backend, which is the only true source: `app.openapi()`. It does **not** commit or publish its own spec. +2. `smartem-frontend` — `packages/api/src/openapi.json`, the Orval input that generates the `@smartem/api` client. +3. `smartem-devtools` — `docs/api/smartem/swagger.json` and `webui/public/api/smartem/swagger.json`, served as human-facing Swagger UI on GitHub Pages and fetched by the frontend's `npm run api:update`. + +Because every copy is refreshed by hand, all three drift. As observed on 2026-06-01, the Pages-served copy was ~9 months stale (25 paths, `0.1.dev276…d20250818`) while the backend served 61 paths; the two devtools copies were themselves inconsistent (25 vs 58 paths). Issue #253 (`smartem-decisions`) records the same drift. A consumer running `npm run api:update` would have regenerated a badly regressed client. + +Two further facts shape the design: + +- **The backend version changes on every commit.** `info.version` is `setuptools_scm`-derived (e.g. `0.1.1rc48.dev3+gcd5206327`), so the *string* changes per commit even when the API surface does not. "Has the API changed?" must therefore be answered by diffing spec **content** (paths + components), not the version field. +- **Compatibility checking exists but is unusable.** ADR 0019 specified a frontend `/version.json` manifest stamping the backend API version it was built against (`write-version-json.mjs`, shipped), plus a boot-time comparison against a backend `/version` endpoint, *observable not enforced*. But the backend `/version` endpoint was never built (only `/status` and `/health` exist), the boot check was never wired into the new app (issue #93), and the helper that does exist (`packages/api/src/version-check.ts`) compares with `serverVersion === API_VERSION` — an **exact full-string match**. Given commit-granular versions, that reports a mismatch on essentially every deployment, so it is wired only into the legacy app and is effectively inert. + +The agent ships from the same repository and tag as the backend, so agent↔backend versions are locked by construction; there is no runtime check, and none is needed. + +## Decision + +Establish a single-source, automated pipeline with the backend as publisher, and finish ADR 0019's compatibility model. + +### 1. The backend is the canonical publisher + +`smartem-decisions` commits its own spec at `docs/api/openapi.json`. A CI job on push to `main` regenerates the spec from `app.openapi()` and, **only when the content has changed** (the spec compared with `info.version` and `servers` normalised out, so per-commit version churn does not trigger it), commits the refreshed file. The committed file is the canonical artefact, fetchable at a stable raw URL on `main`; it is also attached to GitHub Releases for version pinning. `smartem-frontend` and `smartem-devtools` are **downstream caches** of this artefact and are never hand-edited. + +### 2. Backend → devtools sync is push-triggered, and rebuilds Pages + +When the backend commits a changed spec, the same job sends a `repository_dispatch` (`event_type: openapi-spec-updated`) to `smartem-devtools`. A receiver workflow there downloads the canonical spec, writes both devtools copies, and rebuilds GitHub Pages by calling the existing deploy as a reusable job (`workflow_call`) — this side-steps the GitHub rule that a `GITHUB_TOKEN` commit does not itself trigger `on: push` workflows. The workflow also accepts `workflow_dispatch` (manual) and a low-frequency `schedule` as a fallback if a dispatch is ever missed. + +The cross-repo dispatch requires one credential in `smartem-decisions` (a fine-grained token with `Contents: write` on `smartem-devtools`, or a GitHub App). This is the only new secret the design introduces; promptness across a repository boundary is not achievable without one, and a scheduled-only poll was rejected because it does not satisfy the requirement that Pages rebuild *when the backend publishes*. + +### 3. The frontend refreshes from the canonical source + +`smartem-frontend`'s `api:fetch` is repointed from the stale devtools Pages URL to the backend's canonical spec. `packages/api/src/openapi.json` remains committed — it is the hermetic build input for Orval and the source `write-version-json.mjs` stamps `backendApi` from, and it gives a reviewable contract diff in pull requests — but it is now a cache refreshed from the single source. The frontend keeps its own independent semantic version (ADR 0019); that is unaffected. + +### 4. Compatibility is observable, semantic, and finished + +- The backend gains a `GET /version` endpoint returning the API version (the ADR 0019 contract, finally built; `/status` is unchanged). +- `version-check.ts` is rewritten to compare **semantically** — the release portion only, ignoring the `dev`/`+sha` suffix — and to read the backend version from `/version`. It is wired into `apps/smartem/src/main.tsx` to run once, non-blocking, at boot (closing #93). On divergence it logs to the console always and shows a non-blocking banner in development only; production logs. Compatibility is **observed, never enforced**, so rolling updates where the two momentarily differ do not self-inflict an outage. A pinned compatibility range remains deferred, as in ADR 0019. + +### 5. The agent is documented as version-locked + +No runtime check is added; the shared repository and tag guarantee a matched build. This is recorded so the absence of a check is a decision, not an oversight. + +## Consequences + +- Three drifting copies collapse to one source plus two derived caches; the drift class is eliminated and the published Swagger UI tracks the backend automatically. +- `npm run api:update` becomes safe and canonical again. +- One new secret (`DEVTOOLS_DISPATCH_TOKEN`) is required in `smartem-decisions`; the dispatch wiring is inert until it exists, and the manual `workflow_dispatch` path covers the gap. +- This ADR supersedes the relevant surface of ADR 0019: the backend `/version` endpoint and the semantic, observe-only check are now specified and built here. +- Closes `smartem-decisions` #253 (spec sync), `smartem-frontend` #93 (wire the boot check); partially addresses `smartem-devtools` #8 (consolidate API specs). +- New releases are warranted on completion: `smartem-decisions` (new `/version` endpoint and the committed/published spec) and `smartem-frontend` (the wired compatibility check). `smartem-devtools` deploys continuously via Pages and needs no version tag for this change. +- Documentation and Claude Code configuration across the three repositories that described the hand-maintained flow are updated to describe the pipeline. diff --git a/docs/development/tools.md b/docs/development/tools.md index 5961c0a..f2335c8 100644 --- a/docs/development/tools.md +++ b/docs/development/tools.md @@ -168,16 +168,20 @@ uv run python tools/db_table_totals.py ### Generate API Docs -Generates API documentation from OpenAPI specs into `docs/api/`. Processes two APIs: +Generates API documentation from OpenAPI specs into `docs/api/`: -- **Athena API**: copies the original spec from `docs/athena-decision-service-api-spec.json` and adds a local mock server entry -- **SmartEM API**: imports the FastAPI app and extracts the OpenAPI schema at runtime +- **Athena API**: copies the original spec from `docs/athena-decision-service-api-spec.json` and adds a local mock server entry. ```bash uv run python tools/generate_api_docs.py ``` -Output is written to `docs/api/athena/swagger.json` and `docs/api/smartem/swagger.json`. +Output is written to `docs/api/athena/swagger.json`. + +The **SmartEM API** spec is no longer generated here. smartem-decisions is the canonical +publisher (ADR 0020): `docs/api/smartem/swagger.json` and `webui/public/api/smartem/swagger.json` +are downstream caches, refreshed automatically by the `Sync OpenAPI spec from backend` workflow +when smartem-decisions publishes a changed spec (which then rebuilds GitHub Pages). Do not hand-edit them. ## Miscellaneous Tools