diff --git a/CHANGELOG.md b/CHANGELOG.md index ce4fd8b9..74117f95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,41 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `[profile]` extra needs `flux-stats[yaml]>=0.7.0,<0.8`: 0.7.0 is the first flux-stats release that holds these readers and the check below. +### Fixed + +- **A plan made from a `fluid ship --release-dir` release can be applied when the contract has a + dbt build.** The release copied `bundle.tgz` without the provenance file `fluid bundle` writes + beside it, which names the contract the bundle was built from. So a plan made from the release's + bundle recorded no source contract. `fluid apply` then could not find the dbt build's project in + the checkout to check it against the copy the bundle packs, and refused the plan with + `apply_plan_digest_bundle_mismatch` ("the checkout to check it against cannot be found"). + - The release now holds `bundle.tgz.provenance.json`, written for the release's own place: the + same contract, relative to the release directory. A plan made from the release records that + contract, and apply checks the checkout's dbt files, as for a plan made from + `runtime/bundle.tgz`. + - `release.json` lists the file in `files` with its sha256. It stays out of `identity`, which is + unchanged. + - A bundle that names no source contract gets no provenance file in the release, as before, and + neither does a release on another drive. `fluid ship` now warns when that happens + (`release_provenance_missing`). + - The file names the contract relative to where the release was published. So keep the release + at the same place in the checkout, as CI artifacts restored into the same layout are. + +- **A plan can be applied in another checkout of the same tree, as a later CI job may run in + one.** A plan records the planning checkout's absolute paths (`source_contract`, + `source_path`). When the apply job checked the repository out at another path, those paths were + gone, and `fluid apply plan.json --bundle B` refused a contract with a dbt build: it could not + find the dbt project to check ("the plan records source_contract …, which no longer exists"). + - The recorded paths still come first: they are covered by `planDigest`. + - When they are gone, apply anchors the plan at the contract that the bundle's provenance file + names, relative to the bundle. This bundle is the one apply has just verified the plan against. + - The anchor holds for the dbt check, the build runners, relative binding paths and the IaC + source. + - The provenance file is outside every digest. So the contract it names must declare the bundled + contract's id, and the dbt project's files there must still match the copy the bundle packs: + an edit is refused, naming the file. + - With `--no-verify-plan-binding`, nothing is verified, so there is no fallback. + ### Security - **A histogram, a level histogram and a quantile of one column are checked together (FLUX ADR diff --git a/fluid_build/_contract_loader.py b/fluid_build/_contract_loader.py index 385caf6d..db93f385 100644 --- a/fluid_build/_contract_loader.py +++ b/fluid_build/_contract_loader.py @@ -35,12 +35,38 @@ from __future__ import annotations +import contextlib +import contextvars import logging import os import re from importlib import import_module from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, Iterator, List, Optional, Tuple + +#: The bundle the running ``fluid apply plan.json --bundle B`` verified its plan +#: against (the plan's bundle and packet digests matched it), or ``None``. A plan +#: whose recorded checkout paths are gone resolves its source contract through it +#: (:func:`_resolve_plan_source_contract`). Set by :func:`note_applied_bundle`, only +#: inside :func:`applied_bundle_scope`. +_APPLIED_BUNDLE: contextvars.ContextVar[Optional[Path]] = contextvars.ContextVar( + "fluid_applied_bundle", default=None +) + + +@contextlib.contextmanager +def applied_bundle_scope() -> Iterator[None]: + """One ``fluid apply`` run: no verified bundle at its start, none after it returns.""" + token = _APPLIED_BUNDLE.set(None) + try: + yield + finally: + _APPLIED_BUNDLE.reset(token) + + +def note_applied_bundle(bundle: Path) -> None: + """The bundle this apply has just verified its plan against.""" + _APPLIED_BUNDLE.set(Path(bundle)) class CLIError(Exception): @@ -391,7 +417,17 @@ def resolve_source_contract( def _resolve_plan_source_contract(plan_data: Dict[str, Any]) -> Tuple[Optional[Path], str]: """:func:`resolve_source_contract` for a plan: ``contract_metadata.source_contract`` (recorded for plans made from a bundle), else ``source_path``, followed - through the bundle when it names one.""" + through the bundle when it names one, else the bundle this apply verified the + plan against (:data:`_APPLIED_BUNDLE`). + + The recorded paths are the plan job's checkout, under ``planDigest``, so they + win whenever they still exist. They are gone when the apply job checked the + repository out at another path, as one CI job after another may. Then the + verified bundle names its source contract relative to itself, through its + provenance file. That file is outside every digest, so it gets the checks + :func:`resolve_source_contract` gives a bundle (a relative path to a file that + declares the bundled contract's id), and a dbt build's files there are still + compared with the copy the bundle packs.""" meta = plan_data.get("contract_metadata") if isinstance(plan_data, dict) else None if not isinstance(meta, dict): return None, "records no contract_metadata" @@ -409,9 +445,24 @@ def _resolve_plan_source_contract(plan_data: Dict[str, Any]) -> Tuple[Optional[P if resolved is not None: return resolved, "" reasons.append(f"records {key} {recorded}, a bundle that {why}") + through_bundle = _applied_bundle_source(reasons) + if through_bundle is not None: + return through_bundle, "" return None, "; ".join(reasons) or "does not record a source contract" +def _applied_bundle_source(reasons: List[str]) -> Optional[Path]: + """The source contract the bundle this apply verified names, or ``None``, with why + appended to ``reasons`` when there is such a bundle.""" + applied = _APPLIED_BUNDLE.get() + if applied is None: + return None + resolved, why = resolve_source_contract(applied) + if resolved is None: + reasons.append(f"is applied with the bundle {applied}, which {why}") + return resolved + + def source_contract_dir( path: str | Path, logger: Optional[logging.Logger] = None, diff --git a/fluid_build/cli/_ship_release.py b/fluid_build/cli/_ship_release.py index 689e8bbf..750f7279 100644 --- a/fluid_build/cli/_ship_release.py +++ b/fluid_build/cli/_ship_release.py @@ -5,9 +5,9 @@ it ran has passed. The run stages into a sibling ``DIR.partial--*`` directory. When every -selected stage passed, it copies the bundle, the artifacts its MANIFEST lists, -the plan and the reports there, writes ``release.json`` last and renames the -directory to ``DIR``. A run that fails, or is interrupted, publishes nothing: its +selected stage passed, it copies the bundle (with a provenance file that names its +source contract from ``DIR``), the artifacts its MANIFEST lists, the plan and the +reports there, writes ``release.json`` last and renames the directory to ``DIR``. A run that fails, or is interrupted, publishes nothing: its reports stay in ``runtime/``, and a copy in the staging directory, for a reader to inspect. ``DIR`` must not exist, so a release is never changed in place. @@ -26,6 +26,7 @@ import contextlib import hashlib import json +import logging import os import secrets import shutil @@ -36,6 +37,8 @@ from . import _ship_report as report from . import _ship_stages as st +LOG = logging.getLogger(__name__) + RELEASE_SCHEMA = "fluid-release/1" RELEASE_INDEX = "release.json" #: The stages whose output a release holds: they run in the invocation that builds it. @@ -161,8 +164,6 @@ def _bundle_manifest(root: Path) -> Dict[str, Any]: def _emit_set(opts: st.ShipOptions) -> List[str]: """The emit keys stage 3 resolved, in ``generate artifacts``' canonical order.""" - import logging - from fluid_build.forge.core.artifact_fanout import FanoutError, parse_emit_set with st._quiet_probe(): @@ -269,6 +270,53 @@ def _files(staging: Path) -> Dict[str, str]: } +def _write_bundle_provenance( + root: Path, staging: Path, release_dir: Path, bundle_digest: str +) -> None: + """The provenance file beside the release's ``bundle.tgz``, written for where the + release will sit. + + The one ``fluid bundle`` wrote beside ``runtime/bundle.tgz`` names the source + contract relative to ``runtime/``, so a copy would point elsewhere, and a plan made + from the release's bundle would record no source contract: ``fluid apply`` could + then not find a dbt build's project in the checkout to check it against the packed + copy, and refused. The same contract is recorded relative to ``release_dir``, + resolved as its readers resolve the bundle's directory. Like the original, the file + is outside every digest and outside ``identity``, as it depends on where the + release sits; ``release.json`` lists it with the other files. A bundle that names + no usable source contract gets none, and so does a release on another drive. + """ + from fluid_build.forge.core import bundle as bundle_mod + + built = root / st.BUNDLE_PATH + contract, why = bundle_mod.resolve_bundle_source_contract(built) + if contract is None: + _say_no_provenance(f"{st.BUNDLE_PATH} {why}") + return + recorded = bundle_mod.read_bundle_provenance(built) or {} + place = Path(release_dir).parent.resolve() / Path(release_dir).name + try: + contract_rel = Path(os.path.relpath(contract, place)).as_posix() + except ValueError: # another drive on Windows: no relative path exists + _say_no_provenance(f"{release_dir} is on another drive than its source contract {contract}") + return + bundle_mod._write_provenance( + staging / "bundle.tgz", + bundle_digest, + {"contract": contract_rel, "overlay": recorded.get("overlay")}, + ) + + +def _say_no_provenance(why: str) -> None: + """Warn where the release is made: without the file, an apply refuses it later.""" + LOG.warning( + "release_provenance_missing: %s. The release holds no bundle.tgz.provenance.json, so " + "a plan made from its bundle records no source contract, and fluid apply refuses it " + "for a contract with a dbt build.", + why, + ) + + def publish( staging: Path, release_dir: Path, @@ -283,6 +331,7 @@ def publish( rename it to ``release_dir``. On any error nothing is renamed.""" try: _copy(root / st.BUNDLE_PATH, staging / "bundle.tgz") + _write_bundle_provenance(root, staging, release_dir, identity["bundle"]["manifest_digest"]) _copy_artifacts(root, staging) if with_plan: _copy(root / st.PLAN_PATH, staging / "plan.json") diff --git a/fluid_build/cli/apply.py b/fluid_build/cli/apply.py index 52d72a60..a2d9a2c4 100644 --- a/fluid_build/cli/apply.py +++ b/fluid_build/cli/apply.py @@ -1077,8 +1077,13 @@ def _verify_plan_digests( if bundle_path is not None: # The dbt projects the bundle packs: the checkout must still hold them, # where the build runner will read them (its anchor for a plan). - from fluid_build._contract_loader import resolve_source_contract + from fluid_build._contract_loader import note_applied_bundle, resolve_source_contract + # The plan is bound to this bundle, so a plan whose recorded checkout + # paths are gone (planned in another job's checkout) anchors in the + # checkout that holds it, for this check and the rest of the run. Reset + # when the run returns (_scopes_dbt_binding). + note_applied_bundle(Path(bundle_path)) _bind_dbt_projects( plan_data.get("contract") or {}, Path(bundle_path), @@ -1238,13 +1243,15 @@ def _verify_bundle_dbt_projects(args, contract: Dict[str, Any], logger: logging. def _scopes_dbt_binding(fn: Callable[..., int]) -> Callable[..., int]: """One binding scope per ``fluid apply``: its dbt runner re-checks what its own gate bound. - The bound files live in a context variable (``set_bound_projects``); this resets - it when the run returns, so a later apply in the same process starts unbound. + The bound files live in a context variable (``set_bound_projects``), and so does + the bundle the plan was verified against (``applied_bundle_scope``); this resets + both when the run returns, so a later apply in the same process starts unbound. """ import functools @functools.wraps(fn) def _wrapper(args: argparse.Namespace, logger: logging.Logger) -> int: + from fluid_build._contract_loader import applied_bundle_scope from fluid_build.build_runners.dbt.project_files import ( reset_bound_projects, set_bound_projects, @@ -1252,7 +1259,8 @@ def _wrapper(args: argparse.Namespace, logger: logging.Logger) -> int: token = set_bound_projects(None) try: - return fn(args, logger) + with applied_bundle_scope(): + return fn(args, logger) finally: reset_bound_projects(token) diff --git a/tests/cli/test_ship_release.py b/tests/cli/test_ship_release.py index 041d94f2..4c4d4b2e 100644 --- a/tests/cli/test_ship_release.py +++ b/tests/cli/test_ship_release.py @@ -35,6 +35,7 @@ from fluid_build.cli import _ship_report as report from fluid_build.cli import _ship_stages as st from fluid_build.cli import ship +from fluid_build.forge.core import bundle as bundle_mod LOG = logging.getLogger("test_ship_release") REPO_ROOT = Path(fluid_build.__file__).resolve().parent.parent @@ -235,10 +236,13 @@ def test_a_release_names_its_journal_without_an_absolute_path( def test_a_passing_run_publishes_the_release( - work: Path, bundled: Path, monkeypatch: pytest.MonkeyPatch + work: Path, bundled: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: _install(monkeypatch, bundled) - assert _ship([*_UNTIL_4, "--release-dir", "../rel"]) == 0 + with caplog.at_level(logging.WARNING, logger="fluid_build.cli._ship_release"): + assert _ship([*_UNTIL_4, "--release-dir", "../rel"]) == 0 + # The fake bundle stage writes no provenance file, so the release cannot carry one and says so. + assert "release_provenance_missing: runtime/bundle.tgz does not record" in caplog.text rel = work.parent / "rel" assert not list(work.parent.glob("rel.partial-*")) umask = os.umask(0) @@ -259,6 +263,8 @@ def test_a_passing_run_publishes_the_release( for rel_path, digest in index["files"].items(): assert release._sha256(rel / rel_path) == digest assert (rel / "bundle.tgz").read_bytes() == bundled.read_bytes() + # The fake bundle stage writes no provenance file, so the release has none either. + assert not (rel / f"bundle.tgz{bundle_mod.PROVENANCE_SUFFIX}").exists() assert (rel / "artifacts/odcs/product.odcs.orders.yaml").read_text() == "x" assert (rel / "reports/validate-report.json").is_file() shipped = json.loads((rel / "reports/ship-report.json").read_text()) @@ -266,6 +272,40 @@ def test_a_passing_run_publishes_the_release( assert shipped["release"]["identity_digest"] == index["identity_digest"] +def test_the_release_names_its_bundles_source_contract_from_where_it_sits( + work: Path, bundled: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The provenance file beside runtime/bundle.tgz names the contract relative to + runtime/. The release gets its own, relative to the release directory, so a plan + made from the release's bundle anchors at the same contract. It is listed in + ``files`` and stays out of ``identity``.""" + fake = _install(monkeypatch, bundled) + + def bundles_with_provenance(cmd: List[str], **kw: object) -> subprocess.CompletedProcess: + done = fake(cmd, **kw) + if cmd[3] == "bundle": # as fluid bundle --out runtime/bundle.tgz writes it + bundle_mod._write_provenance( + Path(st.BUNDLE_PATH), + release._bundle_manifest(Path.cwd())["digest"], + {"contract": "../contract.fluid.yaml", "overlay": None}, + ) + return done + + monkeypatch.setattr(ship.subprocess, "run", bundles_with_provenance) + assert _ship([*_UNTIL_4, "--release-dir", "../rel"]) == 0 + rel = work.parent / "rel" + name = f"bundle.tgz{bundle_mod.PROVENANCE_SUFFIX}" + index = json.loads((rel / "release.json").read_text()) + assert index["files"][name] == release._sha256(rel / name) + assert bundle_mod.read_bundle_provenance(rel / "bundle.tgz") == { + "contract": "../project/contract.fluid.yaml", + "overlay": None, + } + contract, why = bundle_mod.resolve_bundle_source_contract(rel / "bundle.tgz") + assert (contract, why) == ((work / "contract.fluid.yaml").resolve(), "") + assert name not in json.dumps(index["identity"]) + + def test_a_report_the_checkout_held_is_never_published( work: Path, bundled: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -628,3 +668,139 @@ def test_an_offline_release_run_calls_no_provider_installs_nothing_and_publishes others = [p for p in procs if p[1:3] != ["-m", "fluid_build.cli"]] forbidden = ("pip", "uv", "tofu", "terraform", "dbt", "databricks", "docker", "curl", "git") assert [p for p in others if any(Path(str(p[0])).name.startswith(f) for f in forbidden)] == [] + + +def _dbt_contract() -> Dict[str, object]: + """The release contract, its ``orders`` expose built by a dbt project the bundle packs.""" + contract = _contract() + contract["builds"] = [ + { + "id": "transform", + "pattern": "hybrid-reference", + "engine": "dbt-duckdb", + "repository": "./dbt", + "properties": {"model": "orders"}, + "execution": {"trigger": {"type": "manual"}, "runtime": {"platform": "local"}}, + "outputs": ["orders"], + } + ] + return contract + + +def test_a_plan_from_a_checkout_that_is_gone_resolves_through_the_bundle_it_is_applied_with( + work: Path, bundled: Path +) -> None: + """The plan's recorded paths (under ``planDigest``) win while they exist. When the + plan job's checkout is gone, the bundle this apply verified names the contract, + relative to itself, in this checkout. Nothing is set outside an apply.""" + from fluid_build import _contract_loader as loader + + elsewhere = work.parent / "elsewhere" + gone = { + "contract_metadata": { + "source_contract": str(elsewhere / "contract.fluid.yaml"), + "source_path": str(elsewhere / "runtime" / "bundle.tgz"), + } + } + found, why = loader.resolve_source_contract("plan.json", plan_data=gone) + assert found is None and "which no longer exists" in why + with loader.applied_bundle_scope(): + loader.note_applied_bundle(bundled) + found, why = loader.resolve_source_contract("plan.json", plan_data=gone) + assert (found, why) == ((work / "contract.fluid.yaml").resolve(), "") + recorded = work / "recorded.fluid.yaml" + recorded.write_text(yaml.safe_dump(_contract(), sort_keys=False)) + kept = {"contract_metadata": {"source_contract": str(recorded)}} + assert loader.resolve_source_contract("plan.json", plan_data=kept)[0] == recorded.resolve() + assert loader.resolve_source_contract("plan.json", plan_data=gone)[0] is None + + +def test_apply_notes_the_bundle_it_verified_and_every_anchor_follows_it( + work: Path, bundled: Path +) -> None: + """Once ``fluid apply`` has verified the plan against its bundle, a plan from a + checkout that is gone anchors through that bundle: the binding-path anchor, the + IaC source (which reads plan.json again) and the Spark workspace, as the dbt check + does. The run's scope then clears it.""" + from fluid_build import _contract_loader as loader + from fluid_build.cli import apply as apply_mod + from fluid_build.forge.core.plan_digest import inject_digests + from fluid_build.iac.source import source_contract_file + + elsewhere = work.parent / "elsewhere" + plan = inject_digests( + { + "contract": _contract(), + "contract_metadata": { + "source_contract": str(elsewhere / "contract.fluid.yaml"), + "source_path": str(elsewhere / "runtime" / "bundle.tgz"), + }, + "actions": [], + }, + bundle_path=bundled, + ) + plan_path = work / "plan.json" + plan_path.write_text(json.dumps(plan)) + args = argparse.Namespace(contract=str(plan_path), no_verify_plan_binding=False) + contract = (work / "contract.fluid.yaml").resolve() + with loader.applied_bundle_scope(): + apply_mod._verify_plan_digests(plan, args, LOG, bundle_path=bundled) + assert loader.source_contract_dir(plan_path, plan_data=plan) == contract.parent + assert source_contract_file(plan_path) == contract + assert apply_mod._spark_workspace(args, plan) == contract.parent + assert source_contract_file(plan_path) is None + + +@pytest.mark.integration +def test_a_plan_made_from_the_release_applies_with_its_dbt_project(tmp_path: Path) -> None: + """A plan made from the release's bundle records the contract the bundle was + built from, so ``fluid apply`` finds the checkout's dbt project and checks it + against the packed copy: an unchanged one applies (dry run), an edited one is + refused by name. Before the release carried a provenance file, apply refused + both, as it could not find the checkout (``apply_plan_digest_bundle_mismatch``). + The same holds when the apply runs in another checkout of the tree, as a later + CI job may, and the plan job's checkout is gone.""" + project = tmp_path / "project" + (project / "dbt" / "models").mkdir(parents=True) + (project / "dbt" / "dbt_project.yml").write_text( + "name: orders_dbt\nversion: '1.0.0'\nprofile: orders_dbt\n" + ) + (project / "dbt" / "models" / "orders.sql").write_text("select 1 as id\n") + (project / "contract.fluid.yaml").write_text(yaml.safe_dump(_dbt_contract(), sort_keys=False)) + + def ok(done: subprocess.CompletedProcess) -> str: + assert done.returncode == 0, done.stdout[-3000:] + done.stderr[-3000:] + return done.stdout + done.stderr + + ok( + _cli( + project, + "ship", + "contract.fluid.yaml", + "--until", + "validate-artifacts", + "--no-strict", + "--ci-output", + "plain", + "--release-dir", + "rel", + ) + ) + assert "dbt project bound: transform" in ok( + _cli(project, "plan", "rel/bundle.tgz", "--out", "plans/plan.json") + ) + apply_argv = ("apply", "plans/plan.json", "--bundle", "rel/bundle.tgz", "--dry-run", "--yes") + ok(_cli(project, *apply_argv)) + + import shutil + + # Another job's checkout: the same tree at another path, and the planning one gone. + moved = tmp_path / "another-job" / "project" + shutil.copytree(project, moved, symlinks=True) + project.rename(tmp_path / "planning-checkout-removed") + ok(_cli(moved, *apply_argv)) + + (moved / "dbt" / "models" / "orders.sql").write_text("select 2 as id\n") + edited = _cli(moved, *apply_argv) + assert edited.returncode == 1 + assert "changed models/orders.sql" in edited.stdout + edited.stderr