From 96f36c4e6cf96a83c2abedafb691eda6c06515c7 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 19 Nov 2026 10:57:35 +0100 Subject: [PATCH 1/3] fix(ship): a plan made from a --release-dir release applies a dbt build fluid ship --release-dir copied bundle.tgz without the provenance file fluid bundle writes beside it, so a plan made from the release's bundle recorded no source contract, and fluid apply refused a contract with a dbt build (apply_plan_digest_bundle_mismatch: the checkout to check it against cannot be found). publish() now writes bundle.tgz.provenance.json into the release with the same source contract, relative to the release directory (resolved as its readers resolve the bundle's directory), through bundle._write_provenance. release.json lists it in files; identity is unchanged. The file's format is bundle.py's, which borrows Helm's .tgz.prov pattern: a file beside the archive that names the archive's sha256. --- CHANGELOG.md | 17 ++++++ fluid_build/cli/_ship_release.py | 42 +++++++++++++- tests/cli/test_ship_release.py | 99 ++++++++++++++++++++++++++++++++ 3 files changed, 155 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce4fd8b9..a9681a29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `[profile]` extra needs `flux-stats[yaml]>=0.7.0,<0.8`: 0.7.0 is the first flux-stats release that holds these readers and the check below. +### Fixed + +- **A plan made from a `fluid ship --release-dir` release can be applied when the contract has a + dbt build.** The release copied `bundle.tgz` without the provenance file `fluid bundle` writes + beside it, which names the contract the bundle was built from. So a plan made from the release's + bundle recorded no source contract. `fluid apply` then could not find the dbt build's project in + the checkout to check it against the copy the bundle packs, and refused the plan with + `apply_plan_digest_bundle_mismatch` ("the checkout to check it against cannot be found"). + - The release now holds `bundle.tgz.provenance.json`, written for the release's own place: the + same contract, relative to the release directory. A plan made from the release records that + contract, and apply checks the checkout's dbt files, as for a plan made from + `runtime/bundle.tgz`. + - `release.json` lists the file in `files` with its sha256. It stays out of `identity`, which is + unchanged. + - A bundle that names no source contract gets no provenance file in the release, as before, and + neither does a release on another drive. + ### Security - **A histogram, a level histogram and a quantile of one column are checked together (FLUX ADR diff --git a/fluid_build/cli/_ship_release.py b/fluid_build/cli/_ship_release.py index 689e8bbf..c4a720ea 100644 --- a/fluid_build/cli/_ship_release.py +++ b/fluid_build/cli/_ship_release.py @@ -5,9 +5,9 @@ it ran has passed. The run stages into a sibling ``DIR.partial--*`` directory. When every -selected stage passed, it copies the bundle, the artifacts its MANIFEST lists, -the plan and the reports there, writes ``release.json`` last and renames the -directory to ``DIR``. A run that fails, or is interrupted, publishes nothing: its +selected stage passed, it copies the bundle (with a provenance file that names its +source contract from ``DIR``), the artifacts its MANIFEST lists, the plan and the +reports there, writes ``release.json`` last and renames the directory to ``DIR``. A run that fails, or is interrupted, publishes nothing: its reports stay in ``runtime/``, and a copy in the staging directory, for a reader to inspect. ``DIR`` must not exist, so a release is never changed in place. @@ -269,6 +269,41 @@ def _files(staging: Path) -> Dict[str, str]: } +def _write_bundle_provenance( + root: Path, staging: Path, release_dir: Path, bundle_digest: str +) -> None: + """The provenance file beside the release's ``bundle.tgz``, written for where the + release will sit. + + The one ``fluid bundle`` wrote beside ``runtime/bundle.tgz`` names the source + contract relative to ``runtime/``, so a copy would point elsewhere, and a plan made + from the release's bundle would record no source contract: ``fluid apply`` could + then not find a dbt build's project in the checkout to check it against the packed + copy, and refused. The same contract is recorded relative to ``release_dir``, + resolved as its readers resolve the bundle's directory. Like the original, the file + is outside every digest and outside ``identity``, as it depends on where the + release sits; ``release.json`` lists it with the other files. A bundle that names + no usable source contract gets none, and so does a release on another drive. + """ + from fluid_build.forge.core import bundle as bundle_mod + + built = root / st.BUNDLE_PATH + contract, _why = bundle_mod.resolve_bundle_source_contract(built) + if contract is None: + return + recorded = bundle_mod.read_bundle_provenance(built) or {} + place = Path(release_dir).parent.resolve() / Path(release_dir).name + try: + contract_rel = Path(os.path.relpath(contract, place)).as_posix() + except ValueError: # another drive on Windows: no relative path exists + return + bundle_mod._write_provenance( + staging / "bundle.tgz", + bundle_digest, + {"contract": contract_rel, "overlay": recorded.get("overlay")}, + ) + + def publish( staging: Path, release_dir: Path, @@ -283,6 +318,7 @@ def publish( rename it to ``release_dir``. On any error nothing is renamed.""" try: _copy(root / st.BUNDLE_PATH, staging / "bundle.tgz") + _write_bundle_provenance(root, staging, release_dir, identity["bundle"]["manifest_digest"]) _copy_artifacts(root, staging) if with_plan: _copy(root / st.PLAN_PATH, staging / "plan.json") diff --git a/tests/cli/test_ship_release.py b/tests/cli/test_ship_release.py index 041d94f2..8bba3273 100644 --- a/tests/cli/test_ship_release.py +++ b/tests/cli/test_ship_release.py @@ -35,6 +35,7 @@ from fluid_build.cli import _ship_report as report from fluid_build.cli import _ship_stages as st from fluid_build.cli import ship +from fluid_build.forge.core import bundle as bundle_mod LOG = logging.getLogger("test_ship_release") REPO_ROOT = Path(fluid_build.__file__).resolve().parent.parent @@ -259,6 +260,8 @@ def test_a_passing_run_publishes_the_release( for rel_path, digest in index["files"].items(): assert release._sha256(rel / rel_path) == digest assert (rel / "bundle.tgz").read_bytes() == bundled.read_bytes() + # The fake bundle stage writes no provenance file, so the release has none either. + assert not (rel / f"bundle.tgz{bundle_mod.PROVENANCE_SUFFIX}").exists() assert (rel / "artifacts/odcs/product.odcs.orders.yaml").read_text() == "x" assert (rel / "reports/validate-report.json").is_file() shipped = json.loads((rel / "reports/ship-report.json").read_text()) @@ -266,6 +269,40 @@ def test_a_passing_run_publishes_the_release( assert shipped["release"]["identity_digest"] == index["identity_digest"] +def test_the_release_names_its_bundles_source_contract_from_where_it_sits( + work: Path, bundled: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The provenance file beside runtime/bundle.tgz names the contract relative to + runtime/. The release gets its own, relative to the release directory, so a plan + made from the release's bundle anchors at the same contract. It is listed in + ``files`` and stays out of ``identity``.""" + fake = _install(monkeypatch, bundled) + + def bundles_with_provenance(cmd: List[str], **kw: object) -> subprocess.CompletedProcess: + done = fake(cmd, **kw) + if cmd[3] == "bundle": # as fluid bundle --out runtime/bundle.tgz writes it + bundle_mod._write_provenance( + Path(st.BUNDLE_PATH), + release._bundle_manifest(Path.cwd())["digest"], + {"contract": "../contract.fluid.yaml", "overlay": None}, + ) + return done + + monkeypatch.setattr(ship.subprocess, "run", bundles_with_provenance) + assert _ship([*_UNTIL_4, "--release-dir", "../rel"]) == 0 + rel = work.parent / "rel" + name = f"bundle.tgz{bundle_mod.PROVENANCE_SUFFIX}" + index = json.loads((rel / "release.json").read_text()) + assert index["files"][name] == release._sha256(rel / name) + assert bundle_mod.read_bundle_provenance(rel / "bundle.tgz") == { + "contract": "../project/contract.fluid.yaml", + "overlay": None, + } + contract, why = bundle_mod.resolve_bundle_source_contract(rel / "bundle.tgz") + assert (contract, why) == ((work / "contract.fluid.yaml").resolve(), "") + assert name not in json.dumps(index["identity"]) + + def test_a_report_the_checkout_held_is_never_published( work: Path, bundled: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -628,3 +665,65 @@ def test_an_offline_release_run_calls_no_provider_installs_nothing_and_publishes others = [p for p in procs if p[1:3] != ["-m", "fluid_build.cli"]] forbidden = ("pip", "uv", "tofu", "terraform", "dbt", "databricks", "docker", "curl", "git") assert [p for p in others if any(Path(str(p[0])).name.startswith(f) for f in forbidden)] == [] + + +def _dbt_contract() -> Dict[str, object]: + """The release contract, its ``orders`` expose built by a dbt project the bundle packs.""" + contract = _contract() + contract["builds"] = [ + { + "id": "transform", + "pattern": "hybrid-reference", + "engine": "dbt-duckdb", + "repository": "./dbt", + "properties": {"model": "orders"}, + "execution": {"trigger": {"type": "manual"}, "runtime": {"platform": "local"}}, + "outputs": ["orders"], + } + ] + return contract + + +@pytest.mark.integration +def test_a_plan_made_from_the_release_applies_with_its_dbt_project(tmp_path: Path) -> None: + """A plan made from the release's bundle records the contract the bundle was + built from, so ``fluid apply`` finds the checkout's dbt project and checks it + against the packed copy: an unchanged one applies (dry run), an edited one is + refused by name. Before the release carried a provenance file, apply refused + both, as it could not find the checkout (``apply_plan_digest_bundle_mismatch``).""" + project = tmp_path / "project" + (project / "dbt" / "models").mkdir(parents=True) + (project / "dbt" / "dbt_project.yml").write_text( + "name: orders_dbt\nversion: '1.0.0'\nprofile: orders_dbt\n" + ) + (project / "dbt" / "models" / "orders.sql").write_text("select 1 as id\n") + (project / "contract.fluid.yaml").write_text(yaml.safe_dump(_dbt_contract(), sort_keys=False)) + + def ok(done: subprocess.CompletedProcess) -> str: + assert done.returncode == 0, done.stdout[-3000:] + done.stderr[-3000:] + return done.stdout + done.stderr + + ok( + _cli( + project, + "ship", + "contract.fluid.yaml", + "--until", + "validate-artifacts", + "--no-strict", + "--ci-output", + "plain", + "--release-dir", + "rel", + ) + ) + assert "dbt project bound: transform" in ok( + _cli(project, "plan", "rel/bundle.tgz", "--out", "plans/plan.json") + ) + apply_argv = ("apply", "plans/plan.json", "--bundle", "rel/bundle.tgz", "--dry-run", "--yes") + ok(_cli(project, *apply_argv)) + + (project / "dbt" / "models" / "orders.sql").write_text("select 2 as id\n") + edited = _cli(project, *apply_argv) + assert edited.returncode == 1 + assert "changed models/orders.sql" in edited.stdout + edited.stderr From fd15b656f760511ca4078978a874865621b757c0 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:09:34 +0100 Subject: [PATCH 2/3] fix(apply): a plan applies in another checkout of the same tree A plan records the planning checkout's absolute paths. When a later CI job checks the repository out at another path, fluid apply plan.json --bundle B could not find a dbt build's project and refused the plan. When the recorded paths are gone, the plan now resolves its source contract through the bundle apply has just verified it against (its provenance file, relative to the bundle), for the dbt check, the build runners, binding paths and the IaC source. The recorded paths, under planDigest, still come first. The provenance file is unauthenticated, so resolve_source_contract's id check applies and the dbt files are still compared with the packed copy. Scoped per apply by a context variable, reset when the run returns. Pattern: locate the tree relative to the verified artifact, as Helm's .prov sidecar names files relative to its archive. --- CHANGELOG.md | 17 ++++++++++ fluid_build/_contract_loader.py | 55 +++++++++++++++++++++++++++++++-- fluid_build/cli/apply.py | 16 +++++++--- tests/cli/test_ship_release.py | 44 ++++++++++++++++++++++++-- 4 files changed, 123 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9681a29..f28494d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A bundle that names no source contract gets no provenance file in the release, as before, and neither does a release on another drive. +- **A plan can be applied in another checkout of the same tree, as a later CI job may run in + one.** A plan records the planning checkout's absolute paths (`source_contract`, + `source_path`). When the apply job checked the repository out at another path, those paths were + gone, and `fluid apply plan.json --bundle B` refused a contract with a dbt build: it could not + find the dbt project to check ("the plan records source_contract …, which no longer exists"). + - The recorded paths still come first: they are covered by `planDigest`. + - When they are gone, apply anchors the plan at the contract that the bundle's provenance file + names, relative to the bundle. This bundle is the one apply has just verified the plan against. + - The anchor holds for the dbt check, the build runners, relative binding paths and the IaC + source. + - The provenance file is outside every digest. So the contract it names must declare the bundled + contract's id, and the dbt project's files there must still match the copy the bundle packs: + an edit is refused, naming the file. + - With `--no-verify-plan-binding`, nothing is verified, so there is no fallback. + - GitLab's default checkout path holds the runner and a concurrency slot, so two jobs of one + pipeline can land at different paths; `GIT_CLONE_PATH` is no longer needed for this. + ### Security - **A histogram, a level histogram and a quantile of one column are checked together (FLUX ADR diff --git a/fluid_build/_contract_loader.py b/fluid_build/_contract_loader.py index 385caf6d..db93f385 100644 --- a/fluid_build/_contract_loader.py +++ b/fluid_build/_contract_loader.py @@ -35,12 +35,38 @@ from __future__ import annotations +import contextlib +import contextvars import logging import os import re from importlib import import_module from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, Iterator, List, Optional, Tuple + +#: The bundle the running ``fluid apply plan.json --bundle B`` verified its plan +#: against (the plan's bundle and packet digests matched it), or ``None``. A plan +#: whose recorded checkout paths are gone resolves its source contract through it +#: (:func:`_resolve_plan_source_contract`). Set by :func:`note_applied_bundle`, only +#: inside :func:`applied_bundle_scope`. +_APPLIED_BUNDLE: contextvars.ContextVar[Optional[Path]] = contextvars.ContextVar( + "fluid_applied_bundle", default=None +) + + +@contextlib.contextmanager +def applied_bundle_scope() -> Iterator[None]: + """One ``fluid apply`` run: no verified bundle at its start, none after it returns.""" + token = _APPLIED_BUNDLE.set(None) + try: + yield + finally: + _APPLIED_BUNDLE.reset(token) + + +def note_applied_bundle(bundle: Path) -> None: + """The bundle this apply has just verified its plan against.""" + _APPLIED_BUNDLE.set(Path(bundle)) class CLIError(Exception): @@ -391,7 +417,17 @@ def resolve_source_contract( def _resolve_plan_source_contract(plan_data: Dict[str, Any]) -> Tuple[Optional[Path], str]: """:func:`resolve_source_contract` for a plan: ``contract_metadata.source_contract`` (recorded for plans made from a bundle), else ``source_path``, followed - through the bundle when it names one.""" + through the bundle when it names one, else the bundle this apply verified the + plan against (:data:`_APPLIED_BUNDLE`). + + The recorded paths are the plan job's checkout, under ``planDigest``, so they + win whenever they still exist. They are gone when the apply job checked the + repository out at another path, as one CI job after another may. Then the + verified bundle names its source contract relative to itself, through its + provenance file. That file is outside every digest, so it gets the checks + :func:`resolve_source_contract` gives a bundle (a relative path to a file that + declares the bundled contract's id), and a dbt build's files there are still + compared with the copy the bundle packs.""" meta = plan_data.get("contract_metadata") if isinstance(plan_data, dict) else None if not isinstance(meta, dict): return None, "records no contract_metadata" @@ -409,9 +445,24 @@ def _resolve_plan_source_contract(plan_data: Dict[str, Any]) -> Tuple[Optional[P if resolved is not None: return resolved, "" reasons.append(f"records {key} {recorded}, a bundle that {why}") + through_bundle = _applied_bundle_source(reasons) + if through_bundle is not None: + return through_bundle, "" return None, "; ".join(reasons) or "does not record a source contract" +def _applied_bundle_source(reasons: List[str]) -> Optional[Path]: + """The source contract the bundle this apply verified names, or ``None``, with why + appended to ``reasons`` when there is such a bundle.""" + applied = _APPLIED_BUNDLE.get() + if applied is None: + return None + resolved, why = resolve_source_contract(applied) + if resolved is None: + reasons.append(f"is applied with the bundle {applied}, which {why}") + return resolved + + def source_contract_dir( path: str | Path, logger: Optional[logging.Logger] = None, diff --git a/fluid_build/cli/apply.py b/fluid_build/cli/apply.py index 52d72a60..a2d9a2c4 100644 --- a/fluid_build/cli/apply.py +++ b/fluid_build/cli/apply.py @@ -1077,8 +1077,13 @@ def _verify_plan_digests( if bundle_path is not None: # The dbt projects the bundle packs: the checkout must still hold them, # where the build runner will read them (its anchor for a plan). - from fluid_build._contract_loader import resolve_source_contract + from fluid_build._contract_loader import note_applied_bundle, resolve_source_contract + # The plan is bound to this bundle, so a plan whose recorded checkout + # paths are gone (planned in another job's checkout) anchors in the + # checkout that holds it, for this check and the rest of the run. Reset + # when the run returns (_scopes_dbt_binding). + note_applied_bundle(Path(bundle_path)) _bind_dbt_projects( plan_data.get("contract") or {}, Path(bundle_path), @@ -1238,13 +1243,15 @@ def _verify_bundle_dbt_projects(args, contract: Dict[str, Any], logger: logging. def _scopes_dbt_binding(fn: Callable[..., int]) -> Callable[..., int]: """One binding scope per ``fluid apply``: its dbt runner re-checks what its own gate bound. - The bound files live in a context variable (``set_bound_projects``); this resets - it when the run returns, so a later apply in the same process starts unbound. + The bound files live in a context variable (``set_bound_projects``), and so does + the bundle the plan was verified against (``applied_bundle_scope``); this resets + both when the run returns, so a later apply in the same process starts unbound. """ import functools @functools.wraps(fn) def _wrapper(args: argparse.Namespace, logger: logging.Logger) -> int: + from fluid_build._contract_loader import applied_bundle_scope from fluid_build.build_runners.dbt.project_files import ( reset_bound_projects, set_bound_projects, @@ -1252,7 +1259,8 @@ def _wrapper(args: argparse.Namespace, logger: logging.Logger) -> int: token = set_bound_projects(None) try: - return fn(args, logger) + with applied_bundle_scope(): + return fn(args, logger) finally: reset_bound_projects(token) diff --git a/tests/cli/test_ship_release.py b/tests/cli/test_ship_release.py index 8bba3273..78d1f898 100644 --- a/tests/cli/test_ship_release.py +++ b/tests/cli/test_ship_release.py @@ -684,13 +684,43 @@ def _dbt_contract() -> Dict[str, object]: return contract +def test_a_plan_from_a_checkout_that_is_gone_resolves_through_the_bundle_it_is_applied_with( + work: Path, bundled: Path +) -> None: + """The plan's recorded paths (under ``planDigest``) win while they exist. When the + plan job's checkout is gone, the bundle this apply verified names the contract, + relative to itself, in this checkout. Nothing is set outside an apply.""" + from fluid_build import _contract_loader as loader + + elsewhere = work.parent / "elsewhere" + gone = { + "contract_metadata": { + "source_contract": str(elsewhere / "contract.fluid.yaml"), + "source_path": str(elsewhere / "runtime" / "bundle.tgz"), + } + } + found, why = loader.resolve_source_contract("plan.json", plan_data=gone) + assert found is None and "which no longer exists" in why + with loader.applied_bundle_scope(): + loader.note_applied_bundle(bundled) + found, why = loader.resolve_source_contract("plan.json", plan_data=gone) + assert (found, why) == ((work / "contract.fluid.yaml").resolve(), "") + recorded = work / "recorded.fluid.yaml" + recorded.write_text(yaml.safe_dump(_contract(), sort_keys=False)) + kept = {"contract_metadata": {"source_contract": str(recorded)}} + assert loader.resolve_source_contract("plan.json", plan_data=kept)[0] == recorded.resolve() + assert loader.resolve_source_contract("plan.json", plan_data=gone)[0] is None + + @pytest.mark.integration def test_a_plan_made_from_the_release_applies_with_its_dbt_project(tmp_path: Path) -> None: """A plan made from the release's bundle records the contract the bundle was built from, so ``fluid apply`` finds the checkout's dbt project and checks it against the packed copy: an unchanged one applies (dry run), an edited one is refused by name. Before the release carried a provenance file, apply refused - both, as it could not find the checkout (``apply_plan_digest_bundle_mismatch``).""" + both, as it could not find the checkout (``apply_plan_digest_bundle_mismatch``). + The same holds when the apply runs in another checkout of the tree, as a later + CI job may, and the plan job's checkout is gone.""" project = tmp_path / "project" (project / "dbt" / "models").mkdir(parents=True) (project / "dbt" / "dbt_project.yml").write_text( @@ -723,7 +753,15 @@ def ok(done: subprocess.CompletedProcess) -> str: apply_argv = ("apply", "plans/plan.json", "--bundle", "rel/bundle.tgz", "--dry-run", "--yes") ok(_cli(project, *apply_argv)) - (project / "dbt" / "models" / "orders.sql").write_text("select 2 as id\n") - edited = _cli(project, *apply_argv) + import shutil + + # Another job's checkout: the same tree at another path, and the planning one gone. + moved = tmp_path / "another-job" / "project" + shutil.copytree(project, moved, symlinks=True) + project.rename(tmp_path / "planning-checkout-removed") + ok(_cli(moved, *apply_argv)) + + (moved / "dbt" / "models" / "orders.sql").write_text("select 2 as id\n") + edited = _cli(moved, *apply_argv) assert edited.returncode == 1 assert "changed models/orders.sql" in edited.stdout + edited.stderr From 81c3ff86affab8086f92c9548fb4d45660eab648 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:28:58 +0100 Subject: [PATCH 3/3] fix(ship): warn when a release cannot carry its provenance file A bundle that names no usable source contract, or a release on another drive, still publishes without bundle.tgz.provenance.json, and a later apply of a plan made from it refuses a dbt build. fluid ship now says so where the release is made (release_provenance_missing). Tests now cover the apply side beyond the dbt check: _verify_plan_digests notes the verified bundle, and the binding-path anchor, the IaC source and the Spark workspace follow it while the run's scope holds. The changelog drops an unsourced GitLab claim and says the release must keep its place in the checkout. --- CHANGELOG.md | 7 +++--- fluid_build/cli/_ship_release.py | 19 +++++++++++--- tests/cli/test_ship_release.py | 43 ++++++++++++++++++++++++++++++-- 3 files changed, 61 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f28494d8..74117f95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,7 +46,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `release.json` lists the file in `files` with its sha256. It stays out of `identity`, which is unchanged. - A bundle that names no source contract gets no provenance file in the release, as before, and - neither does a release on another drive. + neither does a release on another drive. `fluid ship` now warns when that happens + (`release_provenance_missing`). + - The file names the contract relative to where the release was published. So keep the release + at the same place in the checkout, as CI artifacts restored into the same layout are. - **A plan can be applied in another checkout of the same tree, as a later CI job may run in one.** A plan records the planning checkout's absolute paths (`source_contract`, @@ -62,8 +65,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 contract's id, and the dbt project's files there must still match the copy the bundle packs: an edit is refused, naming the file. - With `--no-verify-plan-binding`, nothing is verified, so there is no fallback. - - GitLab's default checkout path holds the runner and a concurrency slot, so two jobs of one - pipeline can land at different paths; `GIT_CLONE_PATH` is no longer needed for this. ### Security diff --git a/fluid_build/cli/_ship_release.py b/fluid_build/cli/_ship_release.py index c4a720ea..750f7279 100644 --- a/fluid_build/cli/_ship_release.py +++ b/fluid_build/cli/_ship_release.py @@ -26,6 +26,7 @@ import contextlib import hashlib import json +import logging import os import secrets import shutil @@ -36,6 +37,8 @@ from . import _ship_report as report from . import _ship_stages as st +LOG = logging.getLogger(__name__) + RELEASE_SCHEMA = "fluid-release/1" RELEASE_INDEX = "release.json" #: The stages whose output a release holds: they run in the invocation that builds it. @@ -161,8 +164,6 @@ def _bundle_manifest(root: Path) -> Dict[str, Any]: def _emit_set(opts: st.ShipOptions) -> List[str]: """The emit keys stage 3 resolved, in ``generate artifacts``' canonical order.""" - import logging - from fluid_build.forge.core.artifact_fanout import FanoutError, parse_emit_set with st._quiet_probe(): @@ -288,14 +289,16 @@ def _write_bundle_provenance( from fluid_build.forge.core import bundle as bundle_mod built = root / st.BUNDLE_PATH - contract, _why = bundle_mod.resolve_bundle_source_contract(built) + contract, why = bundle_mod.resolve_bundle_source_contract(built) if contract is None: + _say_no_provenance(f"{st.BUNDLE_PATH} {why}") return recorded = bundle_mod.read_bundle_provenance(built) or {} place = Path(release_dir).parent.resolve() / Path(release_dir).name try: contract_rel = Path(os.path.relpath(contract, place)).as_posix() except ValueError: # another drive on Windows: no relative path exists + _say_no_provenance(f"{release_dir} is on another drive than its source contract {contract}") return bundle_mod._write_provenance( staging / "bundle.tgz", @@ -304,6 +307,16 @@ def _write_bundle_provenance( ) +def _say_no_provenance(why: str) -> None: + """Warn where the release is made: without the file, an apply refuses it later.""" + LOG.warning( + "release_provenance_missing: %s. The release holds no bundle.tgz.provenance.json, so " + "a plan made from its bundle records no source contract, and fluid apply refuses it " + "for a contract with a dbt build.", + why, + ) + + def publish( staging: Path, release_dir: Path, diff --git a/tests/cli/test_ship_release.py b/tests/cli/test_ship_release.py index 78d1f898..4c4d4b2e 100644 --- a/tests/cli/test_ship_release.py +++ b/tests/cli/test_ship_release.py @@ -236,10 +236,13 @@ def test_a_release_names_its_journal_without_an_absolute_path( def test_a_passing_run_publishes_the_release( - work: Path, bundled: Path, monkeypatch: pytest.MonkeyPatch + work: Path, bundled: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: _install(monkeypatch, bundled) - assert _ship([*_UNTIL_4, "--release-dir", "../rel"]) == 0 + with caplog.at_level(logging.WARNING, logger="fluid_build.cli._ship_release"): + assert _ship([*_UNTIL_4, "--release-dir", "../rel"]) == 0 + # The fake bundle stage writes no provenance file, so the release cannot carry one and says so. + assert "release_provenance_missing: runtime/bundle.tgz does not record" in caplog.text rel = work.parent / "rel" assert not list(work.parent.glob("rel.partial-*")) umask = os.umask(0) @@ -712,6 +715,42 @@ def test_a_plan_from_a_checkout_that_is_gone_resolves_through_the_bundle_it_is_a assert loader.resolve_source_contract("plan.json", plan_data=gone)[0] is None +def test_apply_notes_the_bundle_it_verified_and_every_anchor_follows_it( + work: Path, bundled: Path +) -> None: + """Once ``fluid apply`` has verified the plan against its bundle, a plan from a + checkout that is gone anchors through that bundle: the binding-path anchor, the + IaC source (which reads plan.json again) and the Spark workspace, as the dbt check + does. The run's scope then clears it.""" + from fluid_build import _contract_loader as loader + from fluid_build.cli import apply as apply_mod + from fluid_build.forge.core.plan_digest import inject_digests + from fluid_build.iac.source import source_contract_file + + elsewhere = work.parent / "elsewhere" + plan = inject_digests( + { + "contract": _contract(), + "contract_metadata": { + "source_contract": str(elsewhere / "contract.fluid.yaml"), + "source_path": str(elsewhere / "runtime" / "bundle.tgz"), + }, + "actions": [], + }, + bundle_path=bundled, + ) + plan_path = work / "plan.json" + plan_path.write_text(json.dumps(plan)) + args = argparse.Namespace(contract=str(plan_path), no_verify_plan_binding=False) + contract = (work / "contract.fluid.yaml").resolve() + with loader.applied_bundle_scope(): + apply_mod._verify_plan_digests(plan, args, LOG, bundle_path=bundled) + assert loader.source_contract_dir(plan_path, plan_data=plan) == contract.parent + assert source_contract_file(plan_path) == contract + assert apply_mod._spark_workspace(args, plan) == contract.parent + assert source_contract_file(plan_path) is None + + @pytest.mark.integration def test_a_plan_made_from_the_release_applies_with_its_dbt_project(tmp_path: Path) -> None: """A plan made from the release's bundle records the contract the bundle was