From 7b2e2d5bea87416c6f137dd1df932b275df2d4c9 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 19 Nov 2026 09:03:23 +0100 Subject: [PATCH] docs: forge-cli 0.33.0 (pin, contract reference, upgrade notes) --- README.md | 2 +- docs/.vuepress/cli-version.json | 4 +-- docs/README.md | 2 +- docs/advanced/environment-variables.md | 2 +- docs/advanced/operating-in-ci.md | 4 +-- docs/cli/README.md | 2 +- docs/cli/contract-tests.md | 2 +- docs/cli/generate.md | 4 +-- docs/cli/scaffold-ci.md | 2 +- docs/cli/version.md | 2 +- docs/faq/README.md | 2 +- docs/getting-started/README.md | 4 +-- docs/providers/README.md | 2 +- docs/recipes/per-environment-overlays.md | 4 +-- docs/reference/contract-0.7.6.md | 2 +- docs/reference/contract-0.7.7-preview.md | 29 ++++++++++++------- docs/releases.md | 2 +- .../journeys/custom-validator.md | 2 +- .../journeys/your-own-scaffolding.md | 2 +- .../reference/companion-packages.md | 2 +- docs/sdk-and-plugins/reference/trust-model.md | 2 +- docs/upgrading.md | 12 ++++---- docs/vision.md | 2 +- docs/walkthrough/11-stage-pipeline.md | 4 +-- docs/walkthrough/jenkins-cicd.md | 4 +-- docs/walkthrough/local.md | 2 +- scripts/cli-docs-allowlist.yml | 23 --------------- scripts/gen_contract_reference.py | 4 +++ 28 files changed, 59 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index cd6f5b9..0e9d604 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ The site is organised the way a reader uses it (get started, tutorials, how-to g ## Current Versioning -- Current CLI release documented here: `0.32.0` +- Current CLI release documented here: `0.33.0` - Contract schema: `fluidVersion: 0.7.6` is the stable default; `0.7.7` is an opt-in preview Those are different on purpose. `fluid version` reports the installed CLI release, while `fluidVersion` inside a contract selects the contract schema version. diff --git a/docs/.vuepress/cli-version.json b/docs/.vuepress/cli-version.json index 8451ce3..9364e21 100644 --- a/docs/.vuepress/cli-version.json +++ b/docs/.vuepress/cli-version.json @@ -1,6 +1,6 @@ { - "supportedCliVersion": "0.32.0", - "supportedCliInstallSpec": "data-product-forge==0.32.0", + "supportedCliVersion": "0.33.0", + "supportedCliInstallSpec": "data-product-forge==0.33.0", "supportedFluidContractVersion": "0.7.6", "quickstartScaffoldVersion": "0.7.5", "supportedSdkVersion": "0.10.0", diff --git a/docs/README.md b/docs/README.md index e354d07..6a0e31c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -62,7 +62,7 @@ Hello from the my-first-product data product,2026-10-05 10:35:49.479853+02 [Getting Started](./getting-started/README.md) walks through each step with its output, then makes the product read a CSV and adds a data-quality rule. It explains why it uses the `fluid.starter` blueprint rather than `fluid init --quickstart`. -This site documents CLI `0.32.0`, with contract schema `0.7.6` as the stable default and `0.7.7` as an [opt-in preview](./reference/preview-fields.md). `fluid version` is the CLI release you installed; `fluidVersion` inside a contract is the [schema version](./reference/README.md) it is written against. Which version each scaffold writes is in [Understand the version numbers](./getting-started/README.md#understand-the-version-numbers). +This site documents CLI `0.33.0`, with contract schema `0.7.6` as the stable default and `0.7.7` as an [opt-in preview](./reference/preview-fields.md). `fluid version` is the CLI release you installed; `fluidVersion` inside a contract is the [schema version](./reference/README.md) it is written against. Which version each scaffold writes is in [Understand the version numbers](./getting-started/README.md#understand-the-version-numbers). ## Follow one product from laptop to production diff --git a/docs/advanced/environment-variables.md b/docs/advanced/environment-variables.md index 92f0945..81c65cc 100644 --- a/docs/advanced/environment-variables.md +++ b/docs/advanced/environment-variables.md @@ -162,7 +162,7 @@ The CLI itself does not read these; the generated pipeline and the generated DAG | Variable | Read by | Effect | |---|---|---| -| `FLUID_PACKAGE_SPEC` | the generated Jenkins stage 0 | Package spec installed into the workspace venv. Defaults to the generating CLI version with the extras the contract needs, for example `data-product-forge[local]==0.32.0`. | +| `FLUID_PACKAGE_SPEC` | the generated Jenkins stage 0 | Package spec installed into the workspace venv. Defaults to the generating CLI version with the extras the contract needs, for example `data-product-forge[local]==0.33.0`. | | `FLUID_PIP_INDEX_URL`, `FLUID_PIP_EXTRA_INDEX_URL` | the generated install step | Primary and extra pip index. Blank means PyPI. pip takes the highest version across both, so for private packages use one mirror that proxies PyPI in `FLUID_PIP_INDEX_URL` and leave the extra index empty; see [Operating in CI](./operating-in-ci.md#install). | | `FLUID_ALLOW_PRERELEASE` | the generated install step | `true` adds `pip --pre`. | | `FLUID_CONFIG_PATH` | set by generated pipelines | `./fluid_config`. Nothing in the CLI reads it. | diff --git a/docs/advanced/operating-in-ci.md b/docs/advanced/operating-in-ci.md index 530a9a0..caa8761 100644 --- a/docs/advanced/operating-in-ci.md +++ b/docs/advanced/operating-in-ci.md @@ -16,7 +16,7 @@ fluid generate ci contract.fluid.yaml --system jenkins --out Jenkinsfile [install-mode: pypi] Jenkinsfile written -> Jenkinsfile |- Jenkins installs FLUID_PACKAGE_SPEC into $WORKSPACE/.fluid-venv (stage 0) ... - |- FLUID_PACKAGE_SPEC default: data-product-forge[local]==0.32.0 + |- FLUID_PACKAGE_SPEC default: data-product-forge[local]==0.33.0 |- Jenkins plugins required: workflow-aggregator, git ``` @@ -172,7 +172,7 @@ Jenkins learns a job's parameters from its first build. As measured on a Jenkins ### Install -Stage 0 creates `$WORKSPACE/.fluid-venv`, because agents that follow PEP 668 refuse a bare `pip install`, and installs `FLUID_PACKAGE_SPEC` into it. The default is the version that generated the file with the extras the contract's base and overlays need, for example `data-product-forge[local]==0.32.0`. Every stage then runs `fluid` from that venv, and stage 0 fails if the first `fluid` on `PATH` is another one. The agent needs `python3` with the `venv` module, and `rsync` for a `file://` or `ssh://` stage 11 destination. The workspace is removed with Jenkins's core `deleteDir()`, so the `ws-cleanup` plugin is not needed. +Stage 0 creates `$WORKSPACE/.fluid-venv`, because agents that follow PEP 668 refuse a bare `pip install`, and installs `FLUID_PACKAGE_SPEC` into it. The default is the version that generated the file with the extras the contract's base and overlays need, for example `data-product-forge[local]==0.33.0`. Every stage then runs `fluid` from that venv, and stage 0 fails if the first `fluid` on `PATH` is another one. The agent needs `python3` with the `venv` module, and `rsync` for a `file://` or `ssh://` stage 11 destination. The workspace is removed with Jenkins's core `deleteDir()`, so the `ws-cleanup` plugin is not needed. Pin a different version at build time with `FLUID_PACKAGE_SPEC=data-product-forge==X.Y.Z`. `FLUID_PIP_INDEX_URL` (blank means PyPI), `FLUID_PIP_EXTRA_INDEX_URL` and `FLUID_ALLOW_PRERELEASE` (`true` adds `pip --pre`) cover TestPyPI pilots and private mirrors. With `--install-mode dev-source` the file installs from a `/forge-cli-src` bind mount and fails if the mount is missing; it never falls back to PyPI. diff --git a/docs/cli/README.md b/docs/cli/README.md index cfb94ae..b100f46 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -1,6 +1,6 @@ # CLI Reference -This section tracks the command surface of `forge-cli` `0.32.0`, the release `pip install data-product-forge` installs. `fluid --help` shows the promoted commands. +This section tracks the command surface of `forge-cli` `0.33.0`, the release `pip install data-product-forge` installs. `fluid --help` shows the promoted commands. ::: tip Looking up *how to do something* (not *what a command does*)? The page you actually want is [**CLI by task →**](./tasks/) — narrative walkthroughs organized around things you might be trying to accomplish, like "deploy to a new cloud", "add quality rules", "debug a failed run", or "add agent governance". Each task page links to the relevant commands. **This** page is the command index, grouped by purpose — better for lookup, worse for onboarding. diff --git a/docs/cli/contract-tests.md b/docs/cli/contract-tests.md index aff0947..ec3da44 100644 --- a/docs/cli/contract-tests.md +++ b/docs/cli/contract-tests.md @@ -87,7 +87,7 @@ The contract is loaded the way `fluid plan` loads it, with `$ref` fragments and ```yaml - name: Schema gate run: | - pip install data-product-forge==0.32.0 + pip install data-product-forge==0.33.0 fluid contract-tests contract.fluid.yaml --baseline baseline.schema.json ``` diff --git a/docs/cli/generate.md b/docs/cli/generate.md index bfbef5f..35bfcca 100644 --- a/docs/cli/generate.md +++ b/docs/cli/generate.md @@ -528,7 +528,7 @@ Options: | `--no-generate-artifacts` | Skip the transformation and schedule stages, for reference-only contracts. Through 0.32.0 it was also set automatically when any build's `pattern` was `hybrid-reference`, `reference` or `external-reference`. It sets stage 3's default to off on every system. *(since 0.33.0, [forge-cli #1000](https://github.com/Agenticstiger/forge-cli/pull/1000))* Nothing sets it automatically. A contract with reference builds, or only reference builds, still runs stage 3. | | `--install-mode {pypi,dev-source}` | 0.30.1: Jenkins and Tekton. `pypi` (default): the install step installs the package spec (Jenkins: `FLUID_PACKAGE_SPEC` into a virtual environment in the workspace). `dev-source` installs from a `/forge-cli-src` bind mount for contributor labs and fails if the mount is missing; on Tekton it also drops the `fluid-package-spec` and pip index parameters from `pipeline.yaml`. *(since 0.31.0, [forge-cli #942](https://github.com/Agenticstiger/forge-cli/pull/942))* Every system reads it. *(since 0.33.0, [forge-cli #1000](https://github.com/Agenticstiger/forge-cli/pull/1000))* `dev-source` runs `pip install -e /forge-cli-src`, which puts `fluid` on `PATH`, instead of exporting `PYTHONPATH`. On the six other systems it stops with a `FATAL` line when `/forge-cli-src/pyproject.toml` is missing, and when the install fails. The runner needs a package index, or `setuptools` and `setuptools-scm` installed with `PIP_NO_BUILD_ISOLATION=0` (pip reads it inverted). Jenkins keeps the agent's installed `fluid` with `PYTHONPATH=/forge-cli-src`. In 0.32.0 every other system exported `PYTHONPATH=/forge-cli-src` in the install step, which ended with the step and provided no `fluid` command, so the install step failed at `fluid --version`. | | `--image IMAGE` | *(since 0.33.0, [forge-cli #1000](https://github.com/Agenticstiger/forge-cli/pull/1000))* Tekton only: the image the Task steps run. Default `python:3.12-slim`, pinned by digest. It takes `[registry[:port]/]name[:tag][@sha256:]`, and a value that is not an image reference is refused with `generate_ci_invalid_option`. Pin yours by digest too (`name:tag@sha256:`): a tag can move. Another system refuses the flag with `generate_ci_image_unsupported`. | -| `--fluid-package-spec SPEC` | Default of the `FLUID_PACKAGE_SPEC` parameter. Default: the version that generated the file, with the extras the contract's bindings need across its base and every overlay, such as `data-product-forge[local]==0.32.0` or `data-product-forge[aws,local]==0.32.0`. | +| `--fluid-package-spec SPEC` | Default of the `FLUID_PACKAGE_SPEC` parameter. Default: the version that generated the file, with the extras the contract's bindings need across its base and every overlay, such as `data-product-forge[local]==0.33.0` or `data-product-forge[aws,local]==0.33.0`. | | `--fluid-env-default ENV` | Default of `FLUID_ENV`, the overlay the stages pass to `--env`, and the fallback the stages read, so a build with no parameters runs in `ENV`. Letters, digits, `.`, `_` and `-`, starting with a letter. Default `dev`. | | `--apply-mode-default MODE` | Default of `APPLY_MODE`, the mode stage 6 plans and stage 7 applies: `dry-run`, `create-only`, `amend`, `amend-and-build`, `replace` or `replace-and-build`. Default `dry-run` for Jenkins, `amend` for Tekton. | | `--default-publish-target TARGET` | Default of `PUBLISH_TARGETS`, and the fallback stage 10 reads when a build has no parameters. Default `datamesh-manager`. Pick the catalog your team publishes to, for example `fluid-command-center`, `horizon`, `datahub` or `collibra`. | @@ -566,7 +566,7 @@ Below is the 0.30.1 measurement. As measured on 0.18.1 by generating each system *(since 0.31.0, [forge-cli #942](https://github.com/Agenticstiger/forge-cli/pull/942))* Every stage runs `fluid ship "$CONTRACT" --stage N`, and the header of the file says: "A change request (pull request build) stops after stage 6. A branch build applies after a person approves (APPLY_APPROVAL, an input step), one build per environment at a time (lockable-resources)." Stages 7 to 11 sit behind an `Approve apply` stage with an `input` step and a `milestone`, and then run under a `lock` named `fluid-apply-`. They run on the default branch only (`main`, or `--default-branch`), or in a job that builds no branch. `APPLY_APPROVAL` defaults to `true`: set it to `false` for an unattended job. A change request build has no applied-plan baseline for stage 5, because its own job never applies. The sections below describe the 0.30.1 file. -- **Install.** Stage 0 creates `$WORKSPACE/.fluid-venv` and installs `FLUID_PACKAGE_SPEC` into it, because an agent whose system Python is externally managed (PEP 668) refuses a bare `pip install`. The spec defaults to the version that generated the file, so a Jenkinsfile generated by 0.32.0 installs `data-product-forge[local]==0.32.0` and does not move when a new release ships. Override it per build with the `FLUID_PACKAGE_SPEC` parameter, `FLUID_PIP_INDEX_URL` and `FLUID_PIP_EXTRA_INDEX_URL` (TestPyPI), or `FLUID_ALLOW_PRERELEASE`. pip takes the highest version across every index it is given, so do not pair a test or private index with PyPI; see [Operating in CI](../advanced/operating-in-ci.md#install). +- **Install.** Stage 0 creates `$WORKSPACE/.fluid-venv` and installs `FLUID_PACKAGE_SPEC` into it, because an agent whose system Python is externally managed (PEP 668) refuses a bare `pip install`. The spec defaults to the version that generated the file, so a Jenkinsfile generated by 0.33.0 installs `data-product-forge[local]==0.33.0` and does not move when a new release ships. Override it per build with the `FLUID_PACKAGE_SPEC` parameter, `FLUID_PIP_INDEX_URL` and `FLUID_PIP_EXTRA_INDEX_URL` (TestPyPI), or `FLUID_ALLOW_PRERELEASE`. pip takes the highest version across every index it is given, so do not pair a test or private index with PyPI; see [Operating in CI](../advanced/operating-in-ci.md#install). - **Plugins.** `workflow-aggregator` and `git`, plus `copyartifact` with `--diff-last-applied`. *(since 0.31.0, [forge-cli #942](https://github.com/Agenticstiger/forge-cli/pull/942))* Also `lockable-resources` and `junit`. The workspace is removed with the core `deleteDir()` step, so the `ws-cleanup` plugin is not needed. - **The first build.** Jenkins learns a Pipeline's parameters from its first build, so a `buildWithParameters` request to a job that has never run is refused (observed on a Jenkins controller on 4 Oct 2026). Start the job once from the UI with no parameters. The generated file is written for that run: every parameter default is also the fallback each stage reads, so a first build, or the first build after a restart that re-seeds jobs from job-dsl or JCasC, runs what the declarations say. `--fluid-env-default` and `--apply-mode-default` set what that run does. - **Stage 10.** `fluid publish` receives `--env "${FLUID_ENV:-dev}"` by default, so the catalog gets the contract with the overlay stages 5 to 9 used. Before 0.16.3 `fluid publish` had no `--env`, and the default was off. Pass `--no-publish-include-env` only when the pipeline installs a CLI older than `fluid publish --env`. *(since 0.31.0, [forge-cli #942](https://github.com/Agenticstiger/forge-cli/pull/942))* The flag is removed, and ship always passes `--env`. GitLab CI, GitHub Actions, Azure DevOps and Bitbucket pass `--env "${FLUID_ENV:-dev}"` to `fluid publish` in their default output; the default CircleCI output has no `fluid publish` step. diff --git a/docs/cli/scaffold-ci.md b/docs/cli/scaffold-ci.md index cb28b3f..c5ca3e1 100644 --- a/docs/cli/scaffold-ci.md +++ b/docs/cli/scaffold-ci.md @@ -45,7 +45,7 @@ The generated file installs `data-product-forge` from PyPI (`pip install --quiet | publish | On `main` with `DMM_API_URL` set: `fluid publish $CONTRACT --catalog $CATALOG`. | ::: warning Pin the version in a pipeline that holds credentials -An unpinned `pip install data-product-forge` takes whatever release is latest when the job runs, and the same job holds the credentials the apply stage uses. After generating, edit the install line to pin the version you tested, for example `pip install --quiet "data-product-forge==0.32.0"`, and move the pin deliberately when you upgrade. [`fluid generate ci`](./generate.md#fluid-generate-ci) pins the install to the version that generated the file. +An unpinned `pip install data-product-forge` takes whatever release is latest when the job runs, and the same job holds the credentials the apply stage uses. After generating, edit the install line to pin the version you tested, for example `pip install --quiet "data-product-forge==0.33.0"`, and move the pin deliberately when you upgrade. [`fluid generate ci`](./generate.md#fluid-generate-ci) pins the install to the version that generated the file. ::: The settings are `CONTRACT` (the path you passed), `PROVIDER` (default `default`), `BUILD_ID` (default empty; set it to a `builds[].id` to run that build), `CATALOG` (default `datamesh-manager`), `AIRFLOW_DAGS_DEST` and `DMM_API_URL`. GitLab reads them as CI/CD variables, GitHub Actions as `env`, repository `vars` and `secrets`, Jenkins from the `environment` block. diff --git a/docs/cli/version.md b/docs/cli/version.md index 725aadd..e77574a 100644 --- a/docs/cli/version.md +++ b/docs/cli/version.md @@ -69,7 +69,7 @@ Python: 3.11.15 (darwin) System: Darwin arm64 ``` -`fluid --version` is the top-level flag and prints one line, `FLUID Forge CLI v0.32.0`. +`fluid --version` is the top-level flag and prints one line, `FLUID Forge CLI v0.33.0`. ## Options diff --git a/docs/faq/README.md b/docs/faq/README.md index 55c08cb..84a8ea5 100644 --- a/docs/faq/README.md +++ b/docs/faq/README.md @@ -58,7 +58,7 @@ pip install --upgrade "data-product-forge[local]" fluid version ``` -Then follow the [upgrade guide](../upgrading.md). It has a checklist for each version you cross, because some upgrades need action: `0.17.0`, for example, moves remote state to a per-provider key for a contract that uses a per-contract state key, and needs one `fluid schedule-sync` to retire old Airflow DAGs if you sync Airflow DAGs, and `0.16.3` changed some results on purpose. Pin an exact version in CI (`data-product-forge==0.32.0`), not a range. +Then follow the [upgrade guide](../upgrading.md). It has a checklist for each version you cross, because some upgrades need action: `0.17.0`, for example, moves remote state to a per-provider key for a contract that uses a per-contract state key, and needs one `fluid schedule-sync` to retire old Airflow DAGs if you sync Airflow DAGs, and `0.16.3` changed some results on purpose. Pin an exact version in CI (`data-product-forge==0.33.0`), not a range. Your contracts do not need a new `fluidVersion`: `0.27.0` reads `0.7.1` to `0.7.6`, and `0.7.7` as a preview. diff --git a/docs/getting-started/README.md b/docs/getting-started/README.md index 3949bba..45c93cc 100644 --- a/docs/getting-started/README.md +++ b/docs/getting-started/README.md @@ -25,7 +25,7 @@ fluid version ```text ╭───────────────────────────────────── 📦 Version Information ─────────────────────────────────────╮ │ FLUID CLI │ -│ Version: 0.32.0 │ +│ Version: 0.33.0 │ │ API: v1 │ │ │ │ Supported Specifications: │ @@ -55,7 +55,7 @@ The table lists the extras a first project needs. A base install has none of `li ```bash pip install "data-product-forge[local,gcp]" # several extras at once -pip install "data-product-forge[local]==0.32.0" # an exact version, for CI +pip install "data-product-forge[local]==0.33.0" # an exact version, for CI ``` A cloud `fluid apply` runs OpenTofu, so it also needs `tofu` 1.6.0 or newer on your `PATH`, or `fluid apply --ensure-opentofu`, which downloads a pinned build from the OpenTofu release and checks it against that release's `SHA256SUMS` (integrity, not a signature). *(since 0.31.0, [forge-cli #899](https://github.com/Agenticstiger/forge-cli/pull/899))* The minimum is 1.12.0, and the pinned build is 1.13.9. For production CI, install OpenTofu with a cosign- or gpg-verified install instead, and keep `tofu` on the runner image. The local path in this tutorial does not use OpenTofu. diff --git a/docs/providers/README.md b/docs/providers/README.md index cffdcad..f487f9a 100644 --- a/docs/providers/README.md +++ b/docs/providers/README.md @@ -8,7 +8,7 @@ Fluid Forge uses one contract format across local and provider-backed execution ## Docs baseline -- CLI release covered by the primary docs: `0.32.0` +- CLI release covered by the primary docs: `0.33.0` - Contract schema: `0.7.6` is the current stable version, and the default since `0.26.0`; `0.7.7` is a preview you opt into with `fluidVersion: "0.7.7"`. The GCP retention, encryption and `binding.principals` fields need `0.7.6`. - Which `fluidVersion` each scaffolding command writes is listed in [`fluid init`](../cli/init.md#which-fluidversion-each-path-writes). diff --git a/docs/recipes/per-environment-overlays.md b/docs/recipes/per-environment-overlays.md index 78745a3..9fd7359 100644 --- a/docs/recipes/per-environment-overlays.md +++ b/docs/recipes/per-environment-overlays.md @@ -249,14 +249,14 @@ jobs: matrix: env: [dev, staging, prod] steps: - - run: pip install "data-product-forge==0.32.0" + - run: pip install "data-product-forge==0.33.0" - run: fluid validate contract.fluid.yaml --env ${{ matrix.env }} - run: fluid plan contract.fluid.yaml --env ${{ matrix.env }} ``` Pin the version: an unpinned `pip install` in a pipeline takes whatever is newest on the day. Add the extras your clouds need, for example -`"data-product-forge[gcp]==0.32.0"`. +`"data-product-forge[gcp]==0.33.0"`. ## See also diff --git a/docs/reference/contract-0.7.6.md b/docs/reference/contract-0.7.6.md index 2837fa0..5cc024d 100644 --- a/docs/reference/contract-0.7.6.md +++ b/docs/reference/contract-0.7.6.md @@ -9,7 +9,7 @@ description: "The fields of the FLUID 0.7.6 contract schema (stable): path, type Status: **stable**. Schema 0.7.6 is the newest stable schema bundled with the CLI. The newer 0.7.7 schema is a preview; see the [0.7.7 preview delta](./contract-0.7.7-preview.md). -Generated from `fluid_build/schemas/fluid-schema-0.7.6.json` in the `data-product-forge` 0.32.0 package (sha256 `61ca91a63087b89b`). It is the schema `fluid validate` checks a contract against, rendered without edits. To regenerate: `python scripts/gen_contract_reference.py`. +Generated from `fluid_build/schemas/fluid-schema-0.7.6.json` in the `data-product-forge` 0.33.0 package (sha256 `61ca91a63087b89b`). It is the schema `fluid validate` checks a contract against, rendered without edits. To regenerate: `python scripts/gen_contract_reference.py`. Each table row is one field, addressed by its path from the contract root: `.` descends into an object, `[]` marks the items of a list, and `` stands for any key of a map. **Required** means required inside its parent object; the parent can itself be optional. **Allowed values** lists the enum, pattern and range the schema enforces. An object typed plainly `object` rejects keys the schema does not list; `object (open)` accepts extra keys and `object (free-form)` has no listed keys. A row that begins "Only when ..." exists only under that condition. See [how to read this reference](./README.md#how-to-read-a-row). diff --git a/docs/reference/contract-0.7.7-preview.md b/docs/reference/contract-0.7.7-preview.md index 4dc82d3..411694e 100644 --- a/docs/reference/contract-0.7.7-preview.md +++ b/docs/reference/contract-0.7.7-preview.md @@ -9,13 +9,13 @@ description: "What the FLUID 0.7.7 preview schema adds to, and changes in, the s Status: **preview**. A contract opts in with `fluidVersion: "0.7.7"`; a contract that declares 0.7.6 is unaffected. This page lists the difference between the 0.7.6 schema and the 0.7.7 schema as bundled with the CLI: added fields, fields whose definition changed, and fields removed. A field not listed here has the same type, required flag, allowed values, default and description in both; for those, use the [0.7.6 reference](./contract-0.7.6.md). A definition used at several paths is listed once, at its first path. -Generated from `fluid_build/schemas/fluid-schema-0.7.7.json` in the `data-product-forge` 0.32.0 package (sha256 `69980c44e30aef04`). It is the schema `fluid validate` checks a contract against, rendered without edits. To regenerate: `python scripts/gen_contract_reference.py`. +Generated from `fluid_build/schemas/fluid-schema-0.7.7.json` in the `data-product-forge` 0.33.0 package (sha256 `d2eba730d02f43ea`). It is the schema `fluid validate` checks a contract against, rendered without edits. To regenerate: `python scripts/gen_contract_reference.py`. Each table row is one field, addressed by its path from the contract root: `.` descends into an object, `[]` marks the items of a list, and `` stands for any key of a map. **Required** means required inside its parent object; the parent can itself be optional. **Allowed values** lists the enum, pattern and range the schema enforces. An object typed plainly `object` rejects keys the schema does not list; `object (open)` accepts extra keys and `object (free-form)` has no listed keys. A row that begins "Only when ..." exists only under that condition. See [how to read this reference](./README.md#how-to-read-a-row). ## At a glance -- Added: 290 rows +- Added: 297 rows - Changed: 16 rows - Removed: 0 rows @@ -121,19 +121,19 @@ Each table row is one field, addressed by its path from the contract root: `.` d | Path | Type | Required | Allowed values | Default | Description | | --- | --- | --- | --- | --- | --- | -| `builds[].properties.parameters.spark` | object | no | When `runtime` is `emr-eks`: requires `virtualClusterId`
When not (`runtime` is `emr-serverless`): must not set `applicationId`
When not (`runtime` is `emr-serverless`): applicationId is only valid with runtime: emr-serverless
When not (`runtime` is `emr-ec2`): must not set `clusterId`
When not (`runtime` is `emr-ec2`): clusterId is only valid with runtime: emr-ec2
When not (`runtime` is `emr-eks`): must not set `virtualClusterId`
When not (`runtime` is `emr-eks`): virtualClusterId is only valid with runtime: emr-eks
When `runtime` is `glue`: must not set `releaseLabel`
When `runtime` is `glue`: releaseLabel is an EMR release; a glue runtime takes glueVersion
When not (`runtime` is `glue`): must not set `glueVersion`
When not (`runtime` is `glue`): glueVersion is only valid with runtime: glue; EMR runtimes take releaseLabel | | Only when `pattern` is `embedded-logic`. NEW in v0.7.7 (preview): Where and how a Spark build runs. | -| `builds[].properties.parameters.spark.runtime` | string | yes | one of `emr-serverless`, `emr-ec2`, `emr-eks`, `glue` | | Only when `pattern` is `embedded-logic`. The AWS service that runs the job: EMR Serverless, EMR on EC2, EMR on EKS or an AWS Glue Spark job. One runner serves all four. | +| `builds[].properties.parameters.spark` | object | no | When `runtime` is `emr-eks`: requires `virtualClusterId`
When not (`runtime` is one of `emr-serverless`, `local`): must not set `applicationId`
When not (`runtime` is one of `emr-serverless`, `local`): applicationId is only valid with runtime: emr-serverless
When not (`runtime` is one of `emr-ec2`, `local`): must not set `clusterId`
When not (`runtime` is one of `emr-ec2`, `local`): clusterId is only valid with runtime: emr-ec2
When not (`runtime` is one of `emr-eks`, `local`): must not set `virtualClusterId`
When not (`runtime` is one of `emr-eks`, `local`): virtualClusterId is only valid with runtime: emr-eks
When `runtime` is `glue`: must not set `releaseLabel`
When `runtime` is `glue`: releaseLabel is an EMR release; a glue runtime takes glueVersion
When `runtime` is one of `emr-serverless`, `emr-ec2`, `emr-eks`: must not set `glueVersion`
When `runtime` is one of `emr-serverless`, `emr-ec2`, `emr-eks`: glueVersion is only valid with runtime: glue; EMR runtimes take releaseLabel | | Only when `pattern` is `embedded-logic`. NEW in v0.7.7 (preview): Where and how a Spark build runs. | +| `builds[].properties.parameters.spark.runtime` | string | yes | one of `emr-serverless`, `emr-ec2`, `emr-eks`, `glue`, `local` | | Only when `pattern` is `embedded-logic`. Where the job runs: EMR Serverless, EMR on EC2, EMR on EKS, an AWS Glue Spark job, or local, a spark-submit on the machine fluid apply runs on (a laptop, or a lab with an EMR 7.x image). One runner and one job config serve all five. A local run writes a table in an Iceberg REST catalog (location.catalog lakekeeper, rest or polaris) on platform local, which the operator's Spark configuration declares. local ignores the AWS runtimes' keys (applicationId, clusterId, virtualClusterId, releaseLabel, glueVersion, executionRoleArn), so a contract written for AWS switches to local with one overlay line. | | `builds[].properties.parameters.spark.mode` | string | no | one of `streaming`, `batch` | | Only when `pattern` is `embedded-logic`. streaming: a long-running Structured Streaming job. batch: a job that runs to completion. Absent: streaming when properties.stream is set, else batch. | -| `builds[].properties.parameters.spark.applicationId` | string | no | matches `^[0-9a-z]{1,64}(?![\s\S])` | | Only when `pattern` is `embedded-logic`. emr-serverless only: an existing EMR Serverless application. Absent: the forge-managed application shared by the product's domain. | -| `builds[].properties.parameters.spark.clusterId` | string | no | matches `^j-[0-9A-Za-z]{1,254}(?![\s\S])` | | Only when `pattern` is `embedded-logic`. emr-ec2 only: a running EMR cluster (j-...). Absent: a transient cluster that ends with the job. | -| `builds[].properties.parameters.spark.virtualClusterId` | string | no | matches `^[0-9a-z]{1,64}(?![\s\S])` | | Only when `pattern` is `embedded-logic`. emr-eks only, and required there: the EMR on EKS virtual cluster. | -| `builds[].properties.parameters.spark.glueVersion` | string | no | matches `^[0-9]+\.[0-9]+(?![\s\S])` | `"5.0"` | Only when `pattern` is `embedded-logic`. glue only: the AWS Glue version of the job. | -| `builds[].properties.parameters.spark.releaseLabel` | string | no | matches `^emr-[0-9]+\.[0-9]+\.[0-9]+(?![\s\S])` | `"emr-7.12.0"` | Only when `pattern` is `embedded-logic`. EMR runtimes only (emr-serverless, emr-ec2, emr-eks): the EMR release. 7.12 or later writes Iceberg with Lake Formation credentials. | -| `builds[].properties.parameters.spark.executionRoleArn` | string | no | matches `^arn:aws[a-zA-Z0-9-]*:iam::[0-9]{12}:role/[A-Za-z0-9_+=,.@/-]{1,512}(?![\s\S])` | | Only when `pattern` is `embedded-logic`. The IAM role the job runs as. Absent: a least-privilege role forge creates for the build. | +| `builds[].properties.parameters.spark.applicationId` | string | no | matches `^[0-9a-z]{1,64}(?![\s\S])` | | Only when `pattern` is `embedded-logic`. emr-serverless only (runtime local ignores it): an existing EMR Serverless application. Absent: the forge-managed application shared by the product's domain. | +| `builds[].properties.parameters.spark.clusterId` | string | no | matches `^j-[0-9A-Za-z]{1,254}(?![\s\S])` | | Only when `pattern` is `embedded-logic`. emr-ec2 only (runtime local ignores it): a running EMR cluster (j-...). Absent: a transient cluster that ends with the job. | +| `builds[].properties.parameters.spark.virtualClusterId` | string | no | matches `^[0-9a-z]{1,64}(?![\s\S])` | | Only when `pattern` is `embedded-logic`. emr-eks only, and required there (runtime local ignores it): the EMR on EKS virtual cluster. | +| `builds[].properties.parameters.spark.glueVersion` | string | no | matches `^[0-9]+\.[0-9]+(?![\s\S])` | `"5.0"` | Only when `pattern` is `embedded-logic`. glue only (runtime local ignores it): the AWS Glue version of the job. | +| `builds[].properties.parameters.spark.releaseLabel` | string | no | matches `^emr-[0-9]+\.[0-9]+\.[0-9]+((-[a-z0-9]+)*-(latest\|[0-9]{8}))?(?![\s\S])`
When `runtime` is `emr-eks`: matches `^emr-[0-9]+\.[0-9]+\.[0-9]+(-[a-z0-9]+)*-(latest\|[0-9]{8})(?![\s\S])`
When `runtime` is one of `emr-serverless`, `emr-ec2`: matches `^emr-[0-9]+\.[0-9]+\.[0-9]+(?![\s\S])` | `"emr-7.12.0"` | Only when `pattern` is `embedded-logic`. EMR runtimes only (emr-serverless, emr-ec2, emr-eks; runtime local ignores it): the EMR release, in the form the runtime takes. emr-serverless and emr-ec2 name the release (emr-7.12.0, the default); emr-eks names an image of it, emr-X.Y.Z-latest or emr-X.Y.Z-YYYYMMDD (default emr-7.12.0-latest). 7.12 or later writes Iceberg with Lake Formation credentials. | +| `builds[].properties.parameters.spark.executionRoleArn` | string | no | matches `^arn:aws[a-zA-Z0-9-]*:iam::[0-9]{12}:role/[A-Za-z0-9_+=,.@/-]{1,512}(?![\s\S])` | | Only when `pattern` is `embedded-logic`. The IAM role the job runs as (runtime local ignores it: a local job runs with the credentials of the machine it runs on). Absent: a least-privilege role forge creates for the build. | | `builds[].properties.parameters.spark.maskKeySecretArn` | string | no | matches `^arn:aws[a-z-]*:secretsmanager:[a-z0-9-]+:[0-9]{12}:secret:.+(?![\s\S])` | | Only when `pattern` is `embedded-logic`. The AWS Secrets Manager secret holding the HMAC key for maskAtIngest: hash. Required when any mapping column sets maskAtIngest (the cross-field check lives in the Spark compiler). | | `builds[].properties.parameters.spark.sparkConf` | object (map) | no | keys: matches `^[A-Za-z0-9_.-]{1,256}(?![\s\S])` | | Only when `pattern` is `embedded-logic`. Spark configuration properties for the job, name to string value (spark.sql.shuffle.partitions: "64"). | | `builds[].properties.parameters.spark.sparkConf.` | string | no | | | Only when `pattern` is `embedded-logic`. | -| `builds[].properties.parameters.spark.source` | object | no | at least one of `bootstrapServers` or `mskClusterArn` is required
set bootstrapServers (a host:port list) or mskClusterArn (an Amazon MSK cluster)
When `bootstrapServers` is set: must not set `mskClusterArn`
When `bootstrapServers` is set: set one of bootstrapServers or mskClusterArn, not both | | Only when `pattern` is `embedded-logic`. The Kafka connection a streaming Spark build reads, as parameters.flink.source is for a Flink build. Exactly one of bootstrapServers or mskClusterArn. The schema registry is not here: it is properties.stream.registry ($defs/schemaRegistry). | +| `builds[].properties.parameters.spark.source` | object | no | at least one of `bootstrapServers` or `mskClusterArn` is required
set bootstrapServers (a host:port list) or mskClusterArn (an Amazon MSK cluster)
When `bootstrapServers` is set: must not set `mskClusterArn`
When `bootstrapServers` is set: set one of bootstrapServers or mskClusterArn, not both
When `runtime` is `local`: must not set `mskClusterArn`
When `runtime` is `local`: a local run reads source.bootstrapServers: nothing resolves an MSK cluster's brokers for it | | Only when `pattern` is `embedded-logic`. The Kafka connection a streaming Spark build reads, as parameters.flink.source is for a Flink build. Exactly one of bootstrapServers or mskClusterArn. The schema registry is not here: it is properties.stream.registry ($defs/schemaRegistry). | | `builds[].properties.parameters.spark.source.bootstrapServers` | string | no | matches `^\S(?:[^\u0000-\u001f\u007f]*\S)?(?![\s\S])` | | Only when `pattern` is `embedded-logic`. Comma-separated host:port list of Kafka brokers. Same name and shape as parameters.flink.source.bootstrapServers. | | `builds[].properties.parameters.spark.source.mskClusterArn` | string | no | matches `^arn:aws[a-z-]*:kafka:[a-z0-9-]+:[0-9]{12}:cluster/.+(?![\s\S])` | | Only when `pattern` is `embedded-logic`. An Amazon MSK cluster. OpenTofu resolves its brokers through an MSK data source at plan time. | | `builds[].properties.parameters.spark.source.auth` | string | no | one of `msk-iam`, `none` | | Only when `pattern` is `embedded-logic`. How the job authenticates to the brokers. Absent: msk-iam with mskClusterArn, none with bootstrapServers. SASL/SCRAM is not supported yet. | @@ -299,6 +299,13 @@ Each table row is one field, addressed by its path from the contract root: `.` d | `builds[].properties.stream.watermark.delay` | string | yes | matches `^P(?!$)(\d+Y)?(\d+M)?(\d+W)?(\d+D)?(T(\d+H)?(\d+M)?(\d+S)?)?$` | | Only when `pattern` is `embedded-logic`. The allowed lateness, as an ISO-8601 duration (PT10M). | | `builds[].properties.stream.checkpoint` | string | no | matches `^s3://[a-z0-9][a-z0-9.-]{1,61}[a-z0-9](?:/[^\u0000-\u001f\u007f]*)?(?![\s\S])` | | Only when `pattern` is `embedded-logic`. NEW in v0.7.7 (preview): Where the engine keeps its offsets and state, as an s3:// URI. Absent: derived from the product's Iceberg warehouse, one location per product and build. The checkpoint is the job's position in the topic: moving it starts the job again from startingOffsets. | | `builds[].properties.stream.startingOffsets` | string | no | one of `earliest`, `latest` | | Only when `pattern` is `embedded-logic`. NEW in v0.7.7 (preview): Where a job with no checkpoint yet starts reading: earliest (all the topic retains) or latest (new records only). Absent: the engine's default, latest for a Spark stream. A job that has a checkpoint resumes from it. | +| `builds[].properties.stream.deadLetter` | object | no | When `sink` is `topic`: must not set `table`
When `sink` is `topic`: requires `topic`
When `sink` is `topic`: a topic sink names its topic, and no table
When not (`sink` is `topic`): must not set `topic`
When not (`sink` is `topic`): topic is for sink: topic; a table sink takes table | | Only when `pattern` is `embedded-logic`. NEW in v0.7.7 (preview): What forge's Spark runner does with a record it cannot land: a value or key that is not the declared JSON, a field of another type, or a required field that is null (an Avro stream's dead letters are not supported yet). With deadLetter, each such record goes to a dead-letter sink, with its raw bytes, the reason, and its topic, partition and offset, and the stream keeps running; without it, or with enabled: false, the first such record fails the stream. The keys follow the acquisition lane's dlq (enabled, maxRecordsBeforeAbort, alertOn). A Spark stream only: managed Flink refuses the key. | +| `builds[].properties.stream.deadLetter.enabled` | boolean | no | | `true` | Only when `pattern` is `embedded-logic`. false: no dead letters; a record that cannot land fails the stream (an overlay can switch it off per environment). | +| `builds[].properties.stream.deadLetter.sink` | string | no | one of `table`, `topic` | `"table"` | Only when `pattern` is `embedded-logic`. table: an Iceberg table beside the output, in the output's catalog and database. topic: a Kafka topic on the cluster the stream reads, each record with Kafka Connect's error-context headers (__connect.errors.topic, .partition, .offset, .exception.message). | +| `builds[].properties.stream.deadLetter.table` | string | no | matches `^[A-Za-z_][A-Za-z0-9_]{0,254}(?![\s\S])` | | Only when `pattern` is `embedded-logic`. sink table only: the table's name in the output's database. Absent: <output table>__dead_letter, as late events go to <output table>__late_events. | +| `builds[].properties.stream.deadLetter.topic` | string | no | matches `^[A-Za-z0-9._-]{1,249}(?![\s\S])` | | Only when `pattern` is `embedded-logic`. sink topic only, and required there: the dead-letter topic, on the cluster the stream reads. It must exist; the job writes to it and creates nothing. | +| `builds[].properties.stream.deadLetter.maxRecordsBeforeAbort` | integer | no | min 0 | `10000` | Only when `pattern` is `embedded-logic`. The stream fails once it has sent more dead letters than this since it started (0: the first one fails it, after it has landed in the sink). | +| `builds[].properties.stream.deadLetter.alertOn` | array of string | no | unique items
each item: one of `schema_violation` | | Only when `pattern` is `embedded-logic`. The reasons the runner logs a dead_letter_alert line for, each micro-batch that sends any: schema_violation is every record the stream could not land. | | `builds[].properties.mapping` | object | no | | | Only when `pattern` is `embedded-logic`. NEW in v0.7.7 (preview): How the stream's records become the output's rows. Requires stream, and replaces sql. | | `builds[].properties.mapping.explode` | object | no | | | Only when `pattern` is `embedded-logic`. NEW in v0.7.7 (preview): One output row per element of an array. A record whose array is null or empty produces no row. | | `builds[].properties.mapping.explode.path` | string | yes | matches `^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*\|\['[^'\\\u0000-\u001f\u007f]{1,256}'\])*(?![\s\S])` | | Only when `pattern` is `embedded-logic`. The array: a stream column or a nested field, such as items or order.lines. | diff --git a/docs/releases.md b/docs/releases.md index f5d89d9..0637eec 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -5,7 +5,7 @@ description: Every forge-cli release, newest first, with what changed and where # Releases -These docs are verified against CLI `0.32.0`. Moving to a newer CLI? See [Upgrading](./upgrading.md). +These docs are verified against CLI `0.33.0`. Moving to a newer CLI? See [Upgrading](./upgrading.md). Each version links to its upgrade steps, its release notes, or the forge-cli CHANGELOG. For what `fluid version`, `fluidVersion` and the schema versions mean, see [Understand the version numbers](./getting-started/README.md#understand-the-version-numbers). diff --git a/docs/sdk-and-plugins/journeys/custom-validator.md b/docs/sdk-and-plugins/journeys/custom-validator.md index ee40b20..864c5d4 100644 --- a/docs/sdk-and-plugins/journeys/custom-validator.md +++ b/docs/sdk-and-plugins/journeys/custom-validator.md @@ -459,7 +459,7 @@ Three places this typically gets installed: ``` 3. **CI** — the bundle from [your-own-ci](./your-own-ci.md) already has a `validate` stage. Add `my-org-validators` to the `pip install` line: ```yaml - - run: pip install --index-url "" "data-product-forge==0.32.0" "my-org-validators==" + - run: pip install --index-url "" "data-product-forge==0.33.0" "my-org-validators==" - run: fluid validate contract.fluid.yaml --strict env: FLUID_PLUGINS_ALLOWLIST: "steward-required,cost-center-required,classification-from-vocab,local" diff --git a/docs/sdk-and-plugins/journeys/your-own-scaffolding.md b/docs/sdk-and-plugins/journeys/your-own-scaffolding.md index 1a582f0..8834390 100644 --- a/docs/sdk-and-plugins/journeys/your-own-scaffolding.md +++ b/docs/sdk-and-plugins/journeys/your-own-scaffolding.md @@ -176,7 +176,7 @@ keywords = [ dependencies = [ "pydantic>=2.0", - "data-product-forge==0.32.0", + "data-product-forge==0.33.0", ] [project.optional-dependencies] diff --git a/docs/sdk-and-plugins/reference/companion-packages.md b/docs/sdk-and-plugins/reference/companion-packages.md index e58427c..05baf51 100644 --- a/docs/sdk-and-plugins/reference/companion-packages.md +++ b/docs/sdk-and-plugins/reference/companion-packages.md @@ -48,7 +48,7 @@ The PyPI name reflects the product brand (`data-product-forge`). The import path ```toml dependencies = [ - "data-product-forge==0.32.0", # pin exact for reproducibility + "data-product-forge==0.33.0", # pin exact for reproducibility ] ``` diff --git a/docs/sdk-and-plugins/reference/trust-model.md b/docs/sdk-and-plugins/reference/trust-model.md index abdaca9..f4c8c11 100644 --- a/docs/sdk-and-plugins/reference/trust-model.md +++ b/docs/sdk-and-plugins/reference/trust-model.md @@ -177,7 +177,7 @@ Plugins are tested, scanned, and approved by your platform team before showing u ```toml # In your deploy environment's pyproject.toml or requirements: -data-product-forge==0.32.0 +data-product-forge==0.33.0 data-product-forge-sdk==0.10.0 data-product-forge-custom-scaffold==0.4.1 my-org-validators==1.2.0 diff --git a/docs/upgrading.md b/docs/upgrading.md index 197f681..60f8626 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -1,6 +1,6 @@ --- title: Upgrading -description: How to upgrade and pin data-product-forge, the checklist for each step from 0.15.x to 0.32.0, and what moving the local engine from DuckDB 1.x to 2.x changes, including what breaks, what to do, and how to check it. +description: How to upgrade and pin data-product-forge, the checklist for each step from 0.15.x to 0.33.0, and what moving the local engine from DuckDB 1.x to 2.x changes, including what breaks, what to do, and how to check it. --- # Upgrading @@ -19,7 +19,7 @@ fluid version $ fluid version ╭─────────────────────────── 📦 Version Information ───────────────────────────╮ │ FLUID CLI │ -│ Version: 0.32.0 │ +│ Version: 0.33.0 │ │ API: v1 │ │ │ │ Supported Specifications: │ @@ -40,7 +40,7 @@ DuckDB. **Pin an exact version in CI and production:** ```bash -pip install "data-product-forge[local]==0.32.0" +pip install "data-product-forge[local]==0.33.0" ``` A range such as `>=0.16,<0.17` is not safe for CLI behaviour. Patch releases have changed @@ -56,7 +56,7 @@ through the checklist below. ```console $ fluid generate ci contract.fluid.yaml --system jenkins ... - |- FLUID_PACKAGE_SPEC default: data-product-forge[local]==0.32.0 + |- FLUID_PACKAGE_SPEC default: data-product-forge[local]==0.33.0 ``` Upgrading the CLI on your machine does not upgrade that pipeline. Regenerate it, or set @@ -1305,7 +1305,7 @@ These changes are in 0.32.0. Two change what a contract or a generated file look ## Changes in 0.33.0 -These changes are in 0.33.0. The pin on this site stays at 0.32.0 until the pages are checked against that release. +These changes are in 0.33.0. Several change what a plan or a generated file does: read them before you upgrade a pipeline. A BigQuery decimal with a precision replaces its live table once, behind `--allow-data-loss`; a Snowflake column whose type changes needs its table migrated; a Glue table that declares `partitionBy` is replaced the first time it is partitioned; and a generated CI file changes only when you run `fluid generate ci` again. *(since 0.33.0, [forge-cli #965](https://github.com/Agenticstiger/forge-cli/pull/965), [forge-cli #967](https://github.com/Agenticstiger/forge-cli/pull/967))* @@ -1453,7 +1453,7 @@ These changes are in 0.33.0. The pin on this site stays at 0.32.0 until the page ## Not in a release yet -These changes are on forge-cli `main`. The pin on this site stays at 0.32.0 until they ship. +These changes are on forge-cli `main`. The pin on this site is 0.33.0, so they are not in the release these pages document. *On forge-cli `main`, not yet in a release ([forge-cli #1008](https://github.com/Agenticstiger/forge-cli/pull/1008)).* diff --git a/docs/vision.md b/docs/vision.md index eb3ba56..f919f36 100644 --- a/docs/vision.md +++ b/docs/vision.md @@ -149,7 +149,7 @@ fluid market --search "customer analytics" ## Versioning in the docs -- Current CLI release baseline: `0.32.0` +- Current CLI release baseline: `0.33.0` - Current scaffolded contract examples: `fluidVersion: 0.7.5` That split is intentional. The CLI release and the contract schema version move on related but different timelines. diff --git a/docs/walkthrough/11-stage-pipeline.md b/docs/walkthrough/11-stage-pipeline.md index 0586728..7d44c61 100644 --- a/docs/walkthrough/11-stage-pipeline.md +++ b/docs/walkthrough/11-stage-pipeline.md @@ -20,7 +20,7 @@ fluid generate ci contract.fluid.yaml --system jenkins --out Jenkinsfile | FLUID_PIP_INDEX_URL = 'https://test.pypi.org/simple/' (TestPyPI) | FLUID_PIP_EXTRA_INDEX_URL = 'https://pypi.org/simple/' (fallback) | FLUID_ALLOW_PRERELEASE = true (pip --pre, alpha/rc releases) - |- FLUID_PACKAGE_SPEC default: data-product-forge[local]==0.32.0 + |- FLUID_PACKAGE_SPEC default: data-product-forge[local]==0.33.0 |- Jenkins plugins required: workflow-aggregator, git ``` @@ -64,7 +64,7 @@ Jenkins registers a Pipeline job's `parameters` block when the job first runs, s - `APPLY_MODE` is `dry-run`. Stage 6 plans for `dry-run` and stage 7 renders the plan without changing anything. - Stage 8 runs `fluid policy-apply --mode check`, and stage 9 prints that stage 7 ran as a dry run and skips. - Stage 10 (`RUN_STAGE_10_PUBLISH`) and stage 11 (`RUN_STAGE_11_SCHEDULE_SYNC`) are off. Turned on, they print the same dry-run message and skip. -- `CONTRACT` is the path you generated from, `FLUID_ENV` is `dev`, and `FLUID_PACKAGE_SPEC` is the CLI version that generated the file with the extras the contract needs: `data-product-forge[local]==0.32.0` for this contract. +- `CONTRACT` is the path you generated from, `FLUID_ENV` is `dev`, and `FLUID_PACKAGE_SPEC` is the CLI version that generated the file with the extras the contract needs: `data-product-forge[local]==0.33.0` for this contract. Run one plain build first, then use Build with Parameters. Jenkins refuses `buildWithParameters` for a job that has never run, because it has not learned the job's parameters yet (observed on a Jenkins controller on 4 to 5 October 2026 with a pipeline generated by this command). diff --git a/docs/walkthrough/jenkins-cicd.md b/docs/walkthrough/jenkins-cicd.md index 3f92073..80c608a 100644 --- a/docs/walkthrough/jenkins-cicd.md +++ b/docs/walkthrough/jenkins-cicd.md @@ -20,7 +20,7 @@ fluid generate ci contract.fluid.yaml --system jenkins --out Jenkinsfile | FLUID_PIP_INDEX_URL = 'https://test.pypi.org/simple/' (TestPyPI) | FLUID_PIP_EXTRA_INDEX_URL = 'https://pypi.org/simple/' (fallback) | FLUID_ALLOW_PRERELEASE = true (pip --pre, alpha/rc releases) - |- FLUID_PACKAGE_SPEC default: data-product-forge[local]==0.32.0 + |- FLUID_PACKAGE_SPEC default: data-product-forge[local]==0.33.0 |- Jenkins plugins required: workflow-aggregator, git ``` @@ -28,7 +28,7 @@ fluid generate ci contract.fluid.yaml --system jenkins --out Jenkinsfile pip picks the highest version of a package across every index it is given. With `FLUID_PIP_INDEX_URL` set to TestPyPI and `FLUID_PIP_EXTRA_INDEX_URL` set to PyPI, as the output above suggests, whichever index holds the higher version wins, for `data-product-forge` and for each of its dependencies. Anyone can register a name on TestPyPI, so a pilot build can install a package you did not choose. Run TestPyPI pilots only on an agent that holds no deploy credentials. For private packages, set `FLUID_PIP_INDEX_URL` to one mirror that proxies PyPI and leave `FLUID_PIP_EXTRA_INDEX_URL` empty. ::: -Commit the `Jenkinsfile` beside the contract. The file is generated once and then belongs to you: after you upgrade the CLI, regenerate it and review the diff to pick up changes. The `CONTRACT` parameter defaults to the path you generated from, and `FLUID_PACKAGE_SPEC` to the CLI version that generated the file with the extras the contract and its overlays need. A contract that has an `overlays/gcp.yaml` binding to BigQuery generates `data-product-forge[gcp,local]==0.32.0`; `--fluid-package-spec` overrides it. +Commit the `Jenkinsfile` beside the contract. The file is generated once and then belongs to you: after you upgrade the CLI, regenerate it and review the diff to pick up changes. The `CONTRACT` parameter defaults to the path you generated from, and `FLUID_PACKAGE_SPEC` to the CLI version that generated the file with the extras the contract and its overlays need. A contract that has an `overlays/gcp.yaml` binding to BigQuery generates `data-product-forge[gcp,local]==0.33.0`; `--fluid-package-spec` overrides it. ## Set up Jenkins diff --git a/docs/walkthrough/local.md b/docs/walkthrough/local.md index 1704b91..0c451cc 100644 --- a/docs/walkthrough/local.md +++ b/docs/walkthrough/local.md @@ -44,7 +44,7 @@ The `local` extra installs DuckDB (`duckdb>=1.5.0`) and pandas. The bare package ```text ╭───────────────────────────────────── 📦 Version Information ─────────────────────────────────────╮ │ FLUID CLI │ -│ Version: 0.32.0 │ +│ Version: 0.33.0 │ │ API: v1 │ │ │ │ Supported Specifications: │ diff --git a/scripts/cli-docs-allowlist.yml b/scripts/cli-docs-allowlist.yml index 115d97f..c89b967 100644 --- a/scripts/cli-docs-allowlist.yml +++ b/scripts/cli-docs-allowlist.yml @@ -107,29 +107,6 @@ flag_oracle_ok: - docs/cli/forge.md::--resume - docs/cli/forge.md::--fork - docs/cli/forge.md::--seed-from - # On forge-cli main, not in the pinned 0.32.0. Each entry is scoped to the one - # flag; drop it when the pin moves to the release that carries the pull request. - - docs/cli/validate.md::--diagnostics # forge-cli #969 - - docs/cli/ship.md::--require-artifacts # forge-cli #971 - - docs/cli/ship.md::--release-dir # forge-cli #980 - - docs/cli/diff.md::--impact # forge-cli #981 - - docs/cli/diff.md::--plan # forge-cli #981 - - docs/cli/verify.md::--require-every-port # forge-cli #986 - - docs/cli/generate.md::--macros-dir # forge-cli #991 - - docs/cli/generate.md::--coverage # forge-cli #997 - - docs/cli/generate.md::--coverage-out # forge-cli #997 - - docs/cli/generate.md::--require-rule # forge-cli #997 - - docs/cli/generate.md::--require-all-rules # forge-cli #997 - - docs/cli/generate.md::--adapter # forge-cli #997 - - docs/cli/generate.md::--image # forge-cli #1000 - - docs/cli/doctor.md::--contract # forge-cli #992 - - docs/cli/doctor.md::--operation # forge-cli #992 - - docs/cli/doctor.md::--live # forge-cli #992 - - docs/cli/doctor.md::--provider # forge-cli #992 - - docs/cli/doctor.md::--state-backend # forge-cli #992 - - docs/cli/doctor.md::--compare # forge-cli #992 - - docs/cli/ship.md::--journal-dir # forge-cli #1003 - - docs/cli/ship.md::--acknowledge-inspected # forge-cli #1003 # Files the version sweep must not scan, as paths relative to the repo root # (glob syntax allowed). RELEASE_NOTES_*.md is excluded unconditionally and diff --git a/scripts/gen_contract_reference.py b/scripts/gen_contract_reference.py index fd4c96b..f2753f7 100644 --- a/scripts/gen_contract_reference.py +++ b/scripts/gen_contract_reference.py @@ -500,6 +500,10 @@ def describe_if(self, node: Any) -> str: parts.append(f"{code(key)} is {code(_scalar(sub['const']))}") elif "pattern" in sub: parts.append(f"{code(key)} matches {code(sub['pattern'])}") + elif "enum" in sub: + # 0.33.0: `if: {properties: {runtime: {enum: [emr-serverless, local]}}}`. + values = ", ".join(code(_scalar(v)) for v in sub["enum"]) + parts.append(f"{code(key)} is one of {values}") else: die(f"unsupported `if` property schema for {key!r}: {sub!r}") for key in eff.get("required", []):