Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,41 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
`[profile]` extra needs `flux-stats[yaml]>=0.7.0,<0.8`: 0.7.0 is the first flux-stats release
that holds these readers and the check below.

### Fixed

- **A plan made from a `fluid ship --release-dir` release can be applied when the contract has a
dbt build.** The release copied `bundle.tgz` without the provenance file `fluid bundle` writes
beside it, which names the contract the bundle was built from. So a plan made from the release's
bundle recorded no source contract. `fluid apply` then could not find the dbt build's project in
the checkout to check it against the copy the bundle packs, and refused the plan with
`apply_plan_digest_bundle_mismatch` ("the checkout to check it against cannot be found").
- The release now holds `bundle.tgz.provenance.json`, written for the release's own place: the
same contract, relative to the release directory. A plan made from the release records that
contract, and apply checks the checkout's dbt files, as for a plan made from
`runtime/bundle.tgz`.
- `release.json` lists the file in `files` with its sha256. It stays out of `identity`, which is
unchanged.
- A bundle that names no source contract gets no provenance file in the release, as before, and
neither does a release on another drive. `fluid ship` now warns when that happens
(`release_provenance_missing`).
- The file names the contract relative to where the release was published. So keep the release
at the same place in the checkout, as CI artifacts restored into the same layout are.

- **A plan can be applied in another checkout of the same tree, as a later CI job may run in
one.** A plan records the planning checkout's absolute paths (`source_contract`,
`source_path`). When the apply job checked the repository out at another path, those paths were
gone, and `fluid apply plan.json --bundle B` refused a contract with a dbt build: it could not
find the dbt project to check ("the plan records source_contract …, which no longer exists").
- The recorded paths still come first: they are covered by `planDigest`.
- When they are gone, apply anchors the plan at the contract that the bundle's provenance file
names, relative to the bundle. This bundle is the one apply has just verified the plan against.
- The anchor holds for the dbt check, the build runners, relative binding paths and the IaC
source.
- The provenance file is outside every digest. So the contract it names must declare the bundled
contract's id, and the dbt project's files there must still match the copy the bundle packs:
an edit is refused, naming the file.
- With `--no-verify-plan-binding`, nothing is verified, so there is no fallback.

### Security

- **A histogram, a level histogram and a quantile of one column are checked together (FLUX ADR
Expand Down
55 changes: 53 additions & 2 deletions fluid_build/_contract_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,38 @@

from __future__ import annotations

import contextlib
import contextvars
import logging
import os
import re
from importlib import import_module
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from typing import Any, Dict, Iterator, List, Optional, Tuple

#: The bundle the running ``fluid apply plan.json --bundle B`` verified its plan
#: against (the plan's bundle and packet digests matched it), or ``None``. A plan
#: whose recorded checkout paths are gone resolves its source contract through it
#: (:func:`_resolve_plan_source_contract`). Set by :func:`note_applied_bundle`, only
#: inside :func:`applied_bundle_scope`.
_APPLIED_BUNDLE: contextvars.ContextVar[Optional[Path]] = contextvars.ContextVar(
"fluid_applied_bundle", default=None
)


@contextlib.contextmanager
def applied_bundle_scope() -> Iterator[None]:
"""One ``fluid apply`` run: no verified bundle at its start, none after it returns."""
token = _APPLIED_BUNDLE.set(None)
try:
yield
finally:
_APPLIED_BUNDLE.reset(token)


def note_applied_bundle(bundle: Path) -> None:
"""The bundle this apply has just verified its plan against."""
_APPLIED_BUNDLE.set(Path(bundle))


class CLIError(Exception):
Expand Down Expand Up @@ -391,7 +417,17 @@ def resolve_source_contract(
def _resolve_plan_source_contract(plan_data: Dict[str, Any]) -> Tuple[Optional[Path], str]:
""":func:`resolve_source_contract` for a plan: ``contract_metadata.source_contract``
(recorded for plans made from a bundle), else ``source_path``, followed
through the bundle when it names one."""
through the bundle when it names one, else the bundle this apply verified the
plan against (:data:`_APPLIED_BUNDLE`).

The recorded paths are the plan job's checkout, under ``planDigest``, so they
win whenever they still exist. They are gone when the apply job checked the
repository out at another path, as one CI job after another may. Then the
verified bundle names its source contract relative to itself, through its
provenance file. That file is outside every digest, so it gets the checks
:func:`resolve_source_contract` gives a bundle (a relative path to a file that
declares the bundled contract's id), and a dbt build's files there are still
compared with the copy the bundle packs."""
meta = plan_data.get("contract_metadata") if isinstance(plan_data, dict) else None
if not isinstance(meta, dict):
return None, "records no contract_metadata"
Expand All @@ -409,9 +445,24 @@ def _resolve_plan_source_contract(plan_data: Dict[str, Any]) -> Tuple[Optional[P
if resolved is not None:
return resolved, ""
reasons.append(f"records {key} {recorded}, a bundle that {why}")
through_bundle = _applied_bundle_source(reasons)
if through_bundle is not None:
return through_bundle, ""
return None, "; ".join(reasons) or "does not record a source contract"


def _applied_bundle_source(reasons: List[str]) -> Optional[Path]:
"""The source contract the bundle this apply verified names, or ``None``, with why
appended to ``reasons`` when there is such a bundle."""
applied = _APPLIED_BUNDLE.get()
if applied is None:
return None
resolved, why = resolve_source_contract(applied)
if resolved is None:
reasons.append(f"is applied with the bundle {applied}, which {why}")
return resolved


def source_contract_dir(
path: str | Path,
logger: Optional[logging.Logger] = None,
Expand Down
59 changes: 54 additions & 5 deletions fluid_build/cli/_ship_release.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
it ran has passed.

The run stages into a sibling ``DIR.partial-<run-id>-*`` directory. When every
selected stage passed, it copies the bundle, the artifacts its MANIFEST lists,
the plan and the reports there, writes ``release.json`` last and renames the
directory to ``DIR``. A run that fails, or is interrupted, publishes nothing: its
selected stage passed, it copies the bundle (with a provenance file that names its
source contract from ``DIR``), the artifacts its MANIFEST lists, the plan and the
reports there, writes ``release.json`` last and renames the directory to ``DIR``. A run that fails, or is interrupted, publishes nothing: its
reports stay in ``runtime/``, and a copy in the staging directory, for a reader
to inspect. ``DIR`` must not exist, so a release is never changed in place.

Expand All @@ -26,6 +26,7 @@
import contextlib
import hashlib
import json
import logging
import os
import secrets
import shutil
Expand All @@ -36,6 +37,8 @@
from . import _ship_report as report
from . import _ship_stages as st

LOG = logging.getLogger(__name__)

RELEASE_SCHEMA = "fluid-release/1"
RELEASE_INDEX = "release.json"
#: The stages whose output a release holds: they run in the invocation that builds it.
Expand Down Expand Up @@ -161,8 +164,6 @@ def _bundle_manifest(root: Path) -> Dict[str, Any]:

def _emit_set(opts: st.ShipOptions) -> List[str]:
"""The emit keys stage 3 resolved, in ``generate artifacts``' canonical order."""
import logging

from fluid_build.forge.core.artifact_fanout import FanoutError, parse_emit_set

with st._quiet_probe():
Expand Down Expand Up @@ -269,6 +270,53 @@ def _files(staging: Path) -> Dict[str, str]:
}


def _write_bundle_provenance(
root: Path, staging: Path, release_dir: Path, bundle_digest: str
) -> None:
"""The provenance file beside the release's ``bundle.tgz``, written for where the
release will sit.

The one ``fluid bundle`` wrote beside ``runtime/bundle.tgz`` names the source
contract relative to ``runtime/``, so a copy would point elsewhere, and a plan made
from the release's bundle would record no source contract: ``fluid apply`` could
then not find a dbt build's project in the checkout to check it against the packed
copy, and refused. The same contract is recorded relative to ``release_dir``,
resolved as its readers resolve the bundle's directory. Like the original, the file
is outside every digest and outside ``identity``, as it depends on where the
release sits; ``release.json`` lists it with the other files. A bundle that names
no usable source contract gets none, and so does a release on another drive.
"""
from fluid_build.forge.core import bundle as bundle_mod

built = root / st.BUNDLE_PATH
contract, why = bundle_mod.resolve_bundle_source_contract(built)
if contract is None:
_say_no_provenance(f"{st.BUNDLE_PATH} {why}")
return
recorded = bundle_mod.read_bundle_provenance(built) or {}
place = Path(release_dir).parent.resolve() / Path(release_dir).name
try:
contract_rel = Path(os.path.relpath(contract, place)).as_posix()
except ValueError: # another drive on Windows: no relative path exists
_say_no_provenance(f"{release_dir} is on another drive than its source contract {contract}")
return
bundle_mod._write_provenance(
staging / "bundle.tgz",
bundle_digest,
{"contract": contract_rel, "overlay": recorded.get("overlay")},
)


def _say_no_provenance(why: str) -> None:
"""Warn where the release is made: without the file, an apply refuses it later."""
LOG.warning(
"release_provenance_missing: %s. The release holds no bundle.tgz.provenance.json, so "
"a plan made from its bundle records no source contract, and fluid apply refuses it "
"for a contract with a dbt build.",
why,
)


def publish(
staging: Path,
release_dir: Path,
Expand All @@ -283,6 +331,7 @@ def publish(
rename it to ``release_dir``. On any error nothing is renamed."""
try:
_copy(root / st.BUNDLE_PATH, staging / "bundle.tgz")
_write_bundle_provenance(root, staging, release_dir, identity["bundle"]["manifest_digest"])
_copy_artifacts(root, staging)
if with_plan:
_copy(root / st.PLAN_PATH, staging / "plan.json")
Expand Down
16 changes: 12 additions & 4 deletions fluid_build/cli/apply.py
Original file line number Diff line number Diff line change
Expand Up @@ -1077,8 +1077,13 @@ def _verify_plan_digests(
if bundle_path is not None:
# The dbt projects the bundle packs: the checkout must still hold them,
# where the build runner will read them (its anchor for a plan).
from fluid_build._contract_loader import resolve_source_contract
from fluid_build._contract_loader import note_applied_bundle, resolve_source_contract

# The plan is bound to this bundle, so a plan whose recorded checkout
# paths are gone (planned in another job's checkout) anchors in the
# checkout that holds it, for this check and the rest of the run. Reset
# when the run returns (_scopes_dbt_binding).
note_applied_bundle(Path(bundle_path))
_bind_dbt_projects(
plan_data.get("contract") or {},
Path(bundle_path),
Expand Down Expand Up @@ -1238,21 +1243,24 @@ def _verify_bundle_dbt_projects(args, contract: Dict[str, Any], logger: logging.
def _scopes_dbt_binding(fn: Callable[..., int]) -> Callable[..., int]:
"""One binding scope per ``fluid apply``: its dbt runner re-checks what its own gate bound.

The bound files live in a context variable (``set_bound_projects``); this resets
it when the run returns, so a later apply in the same process starts unbound.
The bound files live in a context variable (``set_bound_projects``), and so does
the bundle the plan was verified against (``applied_bundle_scope``); this resets
both when the run returns, so a later apply in the same process starts unbound.
"""
import functools

@functools.wraps(fn)
def _wrapper(args: argparse.Namespace, logger: logging.Logger) -> int:
from fluid_build._contract_loader import applied_bundle_scope
from fluid_build.build_runners.dbt.project_files import (
reset_bound_projects,
set_bound_projects,
)

token = set_bound_projects(None)
try:
return fn(args, logger)
with applied_bundle_scope():
return fn(args, logger)
finally:
reset_bound_projects(token)

Expand Down
Loading
Loading