diff --git a/AGENTS.md b/AGENTS.md index b64c5749..25a0f05c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,7 +49,7 @@ Every contract is delivered through an 11-stage pipeline. Each stage is a hard g | `replace` | auto-snapshot → `CREATE OR REPLACE TABLE` | — | **dropped**; backup retained | | `replace-and-build` | same as `replace` | `dbt run --full-refresh` | **dropped**; rebuilt | -`--mode reconcile [--build-id ]` runs no `tofu`, no DDL and no build, and refuses a `plan.json`: it keeps the contract's always-on Spark streams running with the config the last apply deployed (the job config it uploaded, and the config hash its runs carry). A stream that died starts again on its checkpoint within `execution.retries.maxAttempts` restarts per rolling hour (5 by default, at most 10); drift, a spent budget or nothing deployed starts nothing and exits 1; a run cancelled through the runtime is left stopped. A keepalive (a scheduler task every few minutes, with `--build-id `) keeps a stream up. A plain `fluid apply` restarts a dead stream under the same budget. EMR Serverless and EMR on EKS retry failed runs themselves, so forge never restarts a FAILED run there; Glue's job is emitted with MaxRetries 0, so forge restarts Glue streams as it does EMR on EC2 and local ones. Code: `build_runners/spark/_reconcile.py`, `cli/_spark_apply.py::reconcile_only`. +`--mode reconcile [--build-id ]` runs no `tofu`, no DDL and no build, and refuses a `plan.json`: it keeps the contract's always-on Spark streams running with the config the last apply deployed (the job config it uploaded, and the config hash its runs carry). A stream that died starts again on its checkpoint within `execution.retries.maxAttempts` restarts per rolling hour (5 by default, at most 10); drift, a spent budget or nothing deployed starts nothing and exits 1; a run cancelled through the runtime is left stopped. A keepalive (a scheduler task every few minutes, with `--build-id `) keeps a stream up; `fluid generate schedule` renders one as an Airflow DAG for a stream that declares `execution.retries`. A plain `fluid apply` restarts a dead stream under the same budget. EMR Serverless and EMR on EKS retry failed runs themselves, so forge never restarts a FAILED run there; Glue's job is emitted with MaxRetries 0, so forge restarts Glue streams as it does EMR on EC2 and local ones. Code: `build_runners/spark/_reconcile.py`, `cli/_spark_apply.py::reconcile_only`. `--allow-data-loss` is required for `replace*` when `FLUID_ENV != dev` OR target has rows. `--no-verify-plan-binding` is an emergency DR waiver for plan-binding checks (logged at WARNING level so audit trails catch it). @@ -156,7 +156,7 @@ forge-cli/ | Provider plugins (planning) | `fluid_build/providers/{local,gcp,aws,snowflake,odps,odcs}/` — `plan()` emits abstract actions | | **IaC emitter (apply)** | `fluid_build/iac/` — modular OpenTofu plugin registry. `cli/_apply_opentofu_engine.py` compiles the contract to `main.tf.json` and shells `tofu` for cloud providers. `cli/generate_iac.py` is `fluid generate iac` for review-only emit. One plugin per cloud at `iac/providers/{aws,gcp,snowflake}.py` (dbt-adapter pattern). | | Path-B schedulers | `fluid_build/providers/aws/plan/schedule.py` (EventBridge/MWAA/Lambda/Step-Functions via stage 6 `SchedulePlanner`) | -| Airflow DAGs | `fluid_build/schedulers/airflow/` is the one renderer. `render_dag()` (`__init__.py`, operators in `_operators.py`) renders a contract's `orchestration` section for gcp/aws/snowflake; `fluid_apply.py` renders each build as one `fluid apply --mode amend-and-build --build-id` task, a forge Spark build included (its build phase starts the run and waits; a stream that is not `availableNow` gets no DAG). Builds that `builds[].dependencies` link share one DAG (`scheduled_dags()`: the graph and the schedule; `render_build_task()`: the one function that writes a build's node), on the one trigger they declare: a cron, `trigger.datasets` (`dataset`), or both (`schedule_and_dataset`). Every task declares its build's exposes as Airflow asset `outlets`, named as Airflow's providers name the storage (`_assets.py`). `fluid generate schedule`, `fluid export --engine airflow` and `fluid generate-airflow` (a deprecated alias of the first) all come through it. Task variables go through `codegen_utils.unique_task_identifiers`. | +| Airflow DAGs | `fluid_build/schedulers/airflow/` is the one renderer. `render_dag()` (`__init__.py`, operators in `_operators.py`) renders a contract's `orchestration` section for gcp/aws/snowflake; `fluid_apply.py` renders each build as one `fluid apply --mode amend-and-build --build-id` task, a forge Spark build included (its build phase starts the run and waits; a stream that is not `availableNow` gets no DAG). Builds that `builds[].dependencies` link share one DAG (`scheduled_dags()`: the graph and the schedule; `render_build_task()`: the one function that writes a build's node), on the one trigger they declare: a cron, `trigger.datasets` (`dataset`), or both (`schedule_and_dataset`). Every task declares its build's exposes as Airflow asset `outlets`, named as Airflow's providers name the storage (`_assets.py`). A Spark stream that declares `execution.retries` gets a keepalive DAG whose task runs `fluid apply --mode reconcile --build-id ` every 5 minutes (`KEEPALIVE_APPLY_ARGS`). `fluid generate schedule`, `fluid export --engine airflow` and `fluid generate-airflow` (a deprecated alias of the first) all come through it. Task variables go through `codegen_utils.unique_task_identifiers`. | | Contract schemas | `fluid_build/schemas/*.json` (0.7.2 is latest; `SchemaManager.latest_bundled_version()` is the dynamic lookup) | | Policy engine | `fluid_build/policy/` (compiler, agent_policy, sovereignty, guardrails) | | Plan-binding helpers | `fluid_build/forge/core/plan_digest.py` — `inject_digests`, `verify_plan_binding`, `PlanBindingError` | diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e1fdb5a..0f437477 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `fluid.dead-letter.raw-key` header: the Kafka sink is at least once, and a batch sent again after a failure carries the same keys, so a consumer drops repeats by key. +- **A keepalive DAG for an always-on stream that declares `execution.retries`.** `fluid + generate schedule` gives each forge Spark stream that is not `trigger.availableNow` and + declares `execution.retries` a small DAG whose one task runs + `fluid apply --mode reconcile --build-id --yes`, so a stream a Spot + interruption or a failed driver stopped starts again on its checkpoint. It runs every 5 + minutes, or on the build's `execution.orchestration.airflow.dagConfig.schedule`, one run at a + time, with no catch-up and no Airflow retries: the build's `execution.retries` is the restart + budget, and a run fails when the stream cannot be kept running (it drifted from the contract, + nothing is deployed, its job config cannot be read, its cluster ended, or the restarts are + spent). The worker needs the `FLUID_SPARK_*` settings and the forge release the CI apply has, + or a run can fail too. A run cancelled on purpose stays stopped. The task declares no + outlets, so a dataset trigger on what only the stream writes is refused, as before. A contract + whose only builds are such streams now gets this DAG on Airflow instead of "No build needs a + DAG"; on Prefect or Dagster, and in `fluid scaffold-composer`, it still has nothing to + schedule, and the message names the Airflow keepalive. + ### Fixed - **`fluid apply` runs a dbt build whatever its trigger, finds the dbt that has its adapter, and diff --git a/fluid_build/cli/generate_schedule.py b/fluid_build/cli/generate_schedule.py index 5a5df7e5..0e4d0424 100644 --- a/fluid_build/cli/generate_schedule.py +++ b/fluid_build/cli/generate_schedule.py @@ -170,8 +170,11 @@ def run(args: Any, logger: logging.Logger) -> int: # stream fluid apply keeps running, or a build whose trigger is # continuous) has nothing to schedule, which is not an error: unless it # hand-declares orchestration tasks, or names a scheduler that does not - # exist. - if fluid_apply.runs_only_outside_schedulers(contract) and not ( + # exist. Only the Airflow DAGs keep a stream running (its keepalive DAG), + # so on another scheduler a stream that declares execution.retries has + # nothing to schedule either. + keepalive = (scheduler_name or fluid_apply.DEFAULT_ENGINE) == fluid_apply.DEFAULT_ENGINE + if fluid_apply.runs_only_outside_schedulers(contract, keepalive=keepalive) and not ( fluid_apply.has_explicit_tasks(contract) ): from fluid_build.schedulers import has_scheduler @@ -183,6 +186,11 @@ def run(args: Any, logger: logging.Logger) -> int: "continuously (trigger.type: continuous), so there is nothing to " "schedule. No files written." ) + if not fluid_apply.runs_only_outside_schedulers(contract): + cprint( + "A stream that declares execution.retries gets a keepalive DAG on " + "Airflow: fluid generate schedule --scheduler airflow." + ) return 0 if not scheduler_name: diff --git a/fluid_build/cli/scaffold_composer.py b/fluid_build/cli/scaffold_composer.py index 432597cd..13a2a608 100644 --- a/fluid_build/cli/scaffold_composer.py +++ b/fluid_build/cli/scaffold_composer.py @@ -202,16 +202,23 @@ def _wrote_dag(dag_path: str, dag_id: str) -> bool: ) -def _report_nothing_to_schedule(dag_path: str, dag_id: str) -> None: +def _report_nothing_to_schedule(dag_path: str, dag_id: str, *, keepalive: bool = False) -> None: """Say why no DAG is written, as ``fluid generate schedule`` does, and name a DAG this command wrote for the product before: left in place, it keeps scheduling the product wherever it was deployed. A file of the same name - that it did not write is someone else's, and is not mentioned.""" + that it did not write is someone else's, and is not mentioned. With + ``keepalive``, a stream declares ``execution.retries``: say which command + writes the DAG that keeps it running.""" cprint( "Every build runs outside forge-cli (a Flink job, for example), as a stream that fluid " "apply keeps running, or continuously (trigger.type: continuous), so there is nothing " "to schedule. No DAG written." ) + if keepalive: + cprint( + "A stream that declares execution.retries gets a keepalive DAG from fluid generate " + "schedule, which restarts it when it stops." + ) if _wrote_dag(dag_path, dag_id): # Printed as written: soft_wrap leaves wrapping to the terminal, so the # path is never broken by an inserted newline and copies whole, and no @@ -241,15 +248,18 @@ def run(args, logger: logging.Logger) -> int: # apply keeps running, so a contract with no other build has nothing to # schedule and gets no DAG. The test is `fluid generate schedule`'s, # hand-declared orchestration tasks included, so the two commands agree - # on such a contract. + # on such a contract. A stream's keepalive DAG is generate schedule's + # alone: this command's validate, plan and apply DAG cannot keep a + # stream running, so here a stream with execution.retries is outside too. from fluid_build.schedulers.airflow.fluid_apply import ( has_explicit_tasks, runs_only_outside_schedulers, ) - if runs_only_outside_schedulers(c) and not has_explicit_tasks(c): + if runs_only_outside_schedulers(c, keepalive=False) and not has_explicit_tasks(c): dag_path = os.path.join(os.path.abspath(args.out_dir), f"{dag_id}.py") - _report_nothing_to_schedule(dag_path, dag_id) + kept_alive = not runs_only_outside_schedulers(c) + _report_nothing_to_schedule(dag_path, dag_id, keepalive=kept_alive) return 0 cron = _validate_cron(_extract_cron(c)) # SECURITY (shell + Python-literal injection): the contract path is diff --git a/fluid_build/schedulers/airflow/fluid_apply.py b/fluid_build/schedulers/airflow/fluid_apply.py index 725017b5..5cbdbff0 100644 --- a/fluid_build/schedulers/airflow/fluid_apply.py +++ b/fluid_build/schedulers/airflow/fluid_apply.py @@ -124,6 +124,21 @@ #: destructive mode. APPLY_MODE = "amend-and-build" +#: What a keepalive DAG passes to ``fluid apply`` after the contract and ``--env``, before +#: ``--build-id --yes``. ``--mode reconcile`` runs no ``tofu apply``: it starts that one +#: stream again on its checkpoint when it died, within the restart budget the build's +#: ``execution.retries`` sets, and leaves a run cancelled on purpose stopped. It exits non-zero +#: when it cannot keep the stream running (drift from the contract, nothing deployed, the +#: restarts spent). The one place a keepalive's apply arguments are named. +KEEPALIVE_APPLY_ARGS: Tuple[str, ...] = ("--mode", "reconcile") + +#: How often a keepalive DAG reconciles its stream, unless the build's +#: ``execution.orchestration.airflow.dagConfig.schedule`` names another cron. +KEEPALIVE_SCHEDULE = "*/5 * * * *" + +#: How long one keepalive run may take before Airflow stops it; the next run tries again. +KEEPALIVE_TASK_TIMEOUT_S = 30 * 60 + #: Contract path baked into the DAG when the caller cannot know the real one #: (a bundle carries no source path). Matches fluid's own discovery default #: and the generated CI pipelines' ``CONTRACT`` default. @@ -337,6 +352,8 @@ class ScheduledDag: tasks: Tuple[BuildTask, ...] #: ``(build id, why)``: builds a task depends on that no DAG runs, so none waits for them. unwaited: Tuple[Tuple[str, str], ...] = () + #: A keepalive DAG: its one task reconciles an always-on stream (:data:`KEEPALIVE_APPLY_ARGS`). + keepalive: bool = False # Where a build stands in the schedule (:func:`_classify`). @@ -434,12 +451,76 @@ def _outside_why(build: Dict[str, Any], managed: bool) -> str: return "a job that runs on its own platform (a Flink job, for example)" +def _keeps_alive(build: Dict[str, Any], managed: bool) -> bool: + """Whether a keepalive DAG watches the build: a forge Spark stream (one that is not + ``trigger.availableNow``) that declares ``execution.retries``, on Airflow's engine. A build + another platform runs (:func:`is_unscheduled_build`: ``trigger.type: continuous``, a Flink + job) is that platform's to keep running, as :func:`_classify` has it.""" + if is_unscheduled_build(build) or _spark_kind(build, managed) != "streaming": + return False + execution = build.get("execution") + if not isinstance(execution, dict) or not isinstance(execution.get("retries"), dict): + return False + own = execution.get("orchestration") + return not (isinstance(own, dict) and own.get("engine") not in (None, "", DEFAULT_ENGINE)) + + +def _keepalive_schedule(build: Dict[str, Any]) -> str: + """The cron a keepalive DAG runs on: ``execution.orchestration.airflow.dagConfig.schedule``, + else :data:`KEEPALIVE_SCHEDULE`.""" + own = (build.get("execution") or {}).get("orchestration") + airflow = own.get("airflow") if isinstance(own, dict) else None + config = airflow.get("dagConfig") if isinstance(airflow, dict) else None + value = config.get("schedule") if isinstance(config, dict) else None + if value in (None, ""): + return KEEPALIVE_SCHEDULE + where = f"{build.get('id')!r}: execution.orchestration.airflow.dagConfig.schedule" + try: + cron = _validate_schedule(value, str(build.get("id"))) + except ScheduleRenderError as exc: + raise ScheduleRenderError(f"{where}: {exc}") from None + if cron == "@once": + raise ScheduleRenderError( + f"{where}: a keepalive DAG reconciles the stream on a recurring cron, and @once " + "is not one" + ) + return cron + + def _trigger_schedule(build: Dict[str, Any], managed: bool = True) -> Optional[Dict[str, Any]]: """The build's cron trigger, or ``None`` when it has none Airflow should run.""" kind, trigger = _classify(build, managed) return trigger if kind == _CRON else None +def _keepalive_dags( + contract: Dict[str, Any], builds: List[Dict[str, Any]], managed: bool +) -> List[ScheduledDag]: + """One keepalive DAG per always-on stream that declares ``execution.retries``. + + Its one task runs ``fluid apply --mode reconcile`` for the stream every few minutes, so a + stream a Spot interruption or a failed driver stopped starts again, within the restart + budget the build's ``execution.retries`` sets. A stream is in no build graph (nothing waits + for it), so this reads the builds alone. It has no outlets: a stream updates its tables all + the time, and an asset event on every run would start each DAG scheduled on them. + """ + out: List[ScheduledDag] = [] + for build in builds: + if not _keeps_alive(build, managed): + continue + build_id = _task_build_id(build) + _check_spark_ids(contract, build_id) + out.append( + ScheduledDag( + build_id=build_id, + schedule=DagSchedule(cron=_keepalive_schedule(build), timezone="UTC"), + tasks=(BuildTask(build_id, 0, KEEPALIVE_TASK_TIMEOUT_S),), + keepalive=True, + ) + ) + return out + + def has_scheduled_builds(contract: Dict[str, Any]) -> bool: """True when at least one build declares a trigger a DAG runs on (no validation): a cron, datasets, or both.""" @@ -447,17 +528,28 @@ def has_scheduled_builds(contract: Dict[str, Any]) -> bool: if not isinstance(builds, list): return False managed = _managed_lane(contract) - return any(isinstance(b, dict) and _classify(b, managed)[0] in _DAG_TRIGGERS for b in builds) + return any( + isinstance(b, dict) + and (_classify(b, managed)[0] in _DAG_TRIGGERS or _keeps_alive(b, managed)) + for b in builds + ) -def runs_only_outside_schedulers(contract: Dict[str, Any]) -> bool: +def runs_only_outside_schedulers(contract: Dict[str, Any], *, keepalive: bool = True) -> bool: """True when the contract has builds and no scheduler starts any of them: each is a stream - ``fluid apply`` keeps running, runs continuously, or runs on its own platform.""" + ``fluid apply`` keeps running, runs continuously, or runs on its own platform. + + A stream that declares ``execution.retries`` has a keepalive DAG, so it counts as scheduled. + ``keepalive=False`` is for a renderer that has no keepalive DAG (``scaffold-composer``). + """ builds = [b for b in get_builds(contract) if isinstance(b, dict)] if not builds: return False managed = _managed_lane(contract) - return all(_classify(b, managed)[0] == _OUTSIDE for b in builds) + return all( + _classify(b, managed)[0] == _OUTSIDE and not (keepalive and _keeps_alive(b, managed)) + for b in builds + ) def has_explicit_tasks(contract: Dict[str, Any]) -> bool: @@ -974,12 +1066,20 @@ def _expose_ref_uri(self, where: str, ref: Mapping[str, Any]) -> str: def check_expose_refs(self, dags: List[ScheduledDag]) -> None: """Refuse a dataset trigger on an expose of this contract that no task updates: every - build that writes it runs outside any DAG, so the trigger would never fire.""" - tasked = {t.build_id for dag in dags for t in dag.tasks} + build that writes it runs outside any DAG, or only in a keepalive DAG, which declares no + outlets, so the trigger would never fire.""" + tasked = {t.build_id for dag in dags if not dag.keepalive for t in dag.tasks} for where, expose_id, writers in self.expose_refs: if not writers or any(_expose_writer_id(b) in tasked for b in writers): continue names = ", ".join(repr(b.get("id")) for b in writers) + if all(_keeps_alive(b, self.managed) for b in writers): + raise ScheduleRenderError( + f"{where}: expose {expose_id!r} is written by {names}, whose keepalive DAG " + "declares no outlets (a stream updates its tables all the time), so no task " + "ever updates its asset and the trigger would never fire. Run this build on " + "a schedule (trigger.type: schedule)" + ) raise ScheduleRenderError( f"{where}: expose {expose_id!r} is written by {names}, which no DAG runs " f"({self._why_no_task(writers[0])}), so no task ever updates its asset and the " @@ -1055,6 +1155,7 @@ def scheduled_dags(contract: Dict[str, Any]) -> List[ScheduledDag]: return [] graph = _BuildGraph(contract, [b for b in raw_builds if isinstance(b, dict)]) dags = [dag for members in graph.groups() if (dag := graph.dag(members)) is not None] + dags += _keepalive_dags(contract, graph.builds, graph.managed) seen: Set[str] = set() for task in (t for dag in dags for t in dag.tasks): if task.build_id in seen: @@ -1279,6 +1380,21 @@ def rendered_dag_of(path: Path) -> Optional[Tuple[str, str]]: BASH_SCRIPT_LINES = _SCRIPT_HEAD + _APPLY_TAIL + _SCRIPT_END +def _apply_args(args: Tuple[str, ...], build_ref: str) -> str: + """``args``, then ``--build-id --yes``: one space apart, however many ``args``.""" + return " ".join((*args, "--build-id", build_ref, "--yes")) + + +_KEEPALIVE_TAIL = ( + 'set -- apply "$contract"', + 'if [ -n "$FLUID_DAG_ENV" ]; then set -- "$@" --env "$FLUID_DAG_ENV"; fi', + 'set -- "$@" ' + _apply_args(KEEPALIVE_APPLY_ARGS, '"$FLUID_DAG_BUILD_ID"'), +) + +#: The script of a keepalive DAG's task: :data:`BASH_SCRIPT_LINES` but for the apply arguments. +KEEPALIVE_SCRIPT_LINES = _SCRIPT_HEAD + _KEEPALIVE_TAIL + _SCRIPT_END + + def _wrap(words: List[str], indent: str, width: int = 76) -> str: lines: List[str] = [] current = indent @@ -1317,6 +1433,26 @@ def _wrap(words: List[str], indent: str, width: int = 76) -> str: ' --mode amend-and-build --build-id "$BUILD_ID" --yes\n' ) +_KEEPALIVE_RUN = ( + "Every run reconciles one always-on stream on the Airflow worker, with the\n" + "constants below filled in (`--env` is left out when FLUID_ENV_NAME is empty):\n" + "\n" + ' cd "$FLUID_PROJECT_DIR"\n' + ' fluid apply "$FLUID_PROJECT_DIR/$CONTRACT_PATH" --env "$FLUID_ENV_NAME" \\\\\n' + " " + _apply_args(KEEPALIVE_APPLY_ARGS, '"$BUILD_ID"') + "\n" + "\n" + "It runs no tofu apply. When the stream died (a Spot interruption, a failed\n" + "driver), it starts again on its checkpoint, within the restart budget the\n" + "build's execution.retries sets, so the task itself never retries. The run\n" + "fails when the stream cannot be kept running: it drifted from the contract,\n" + "nothing is deployed, the job config the last apply deployed cannot be read,\n" + "its cluster ended, or the restarts are spent. It can also fail when this\n" + "worker's FLUID_SPARK_* settings or forge release differ from the CI apply's.\n" + "A run cancelled on purpose stays stopped until fluid apply starts it. It\n" + "runs on SCHEDULE, one run at a time, with no catch-up. The build's\n" + "execution.orchestration.airflow.dagConfig.schedule sets SCHEDULE.\n" +) + _BUILDS_RUN = ( "Each task runs one build on the Airflow worker, with the constants below\n" "filled in (`--env` is left out when FLUID_ENV_NAME is empty) and the\n" @@ -1447,7 +1583,7 @@ def _schedule_expr(schedule: DagSchedule) -> str: ) -def render_build_task(task: BuildTask, *, only: bool) -> str: +def render_build_task(task: BuildTask, *, only: bool, keepalive: bool = False) -> str: """The source of one build's node: a BashOperator that runs ``fluid apply --build-id``. This is the one function that decides what runs a build; the DAG's schedule and its edges @@ -1458,7 +1594,8 @@ def render_build_task(task: BuildTask, *, only: bool) -> str: """ lit = py_str_literal if only: - head, task_id, build_id = " BashOperator(\n", '"fluid_apply"', "BUILD_ID" + name = '"fluid_keepalive"' if keepalive else '"fluid_apply"' + head, task_id, build_id = " BashOperator(\n", name, "BUILD_ID" else: head = f" tasks[{lit(task.build_id)}] = BashOperator(\n" task_id = build_id = lit(task.build_id) @@ -1523,6 +1660,11 @@ def _edges(tasks: Tuple[BuildTask, ...]) -> str: def _dag_doc(product_id: str, dag: ScheduledDag) -> Tuple[str, str]: """The DAG file's docstring and the DAG's ``description``.""" + if dag.keepalive: + return ( + _doc("Scheduled fluid build", _KEEPALIVE_RUN, []), + f"keep fluid stream {product_id} {dag.build_id} running", + ) only = len(dag.tasks) == 1 notes = [] if only else [_TRIGGER_RULES_NOTE] notes += [n for n in (_schedule_note(dag.schedule), _unwaited_note(dag.unwaited)) if n] @@ -1560,7 +1702,7 @@ def _dag_tasks(dag: ScheduledDag) -> Tuple[str, str]: ' "retry_delay": timedelta(minutes=5),\n' f' "execution_timeout": {_timeout_expr(task.task_timeout_s)},\n' ) - return default_args, render_build_task(task, only=True) + return default_args, render_build_task(task, only=True, keepalive=dag.keepalive) nodes = "".join(render_build_task(t, only=False) for t in dag.tasks) default_args = ' "owner": "fluid",\n "retry_delay": timedelta(minutes=5),\n' return default_args, f" tasks = {{}}\n{nodes}\n{_edges(dag.tasks)}" @@ -1579,7 +1721,8 @@ def render_scheduled_dag( doc, description = _dag_doc(product_id, dag) default_args, body = _dag_tasks(dag) uses_assets = bool(dag.schedule.datasets) or any(t.outlets for t in dag.tasks) - script = "".join(f" {lit(line)},\n" for line in BASH_SCRIPT_LINES) + lines = KEEPALIVE_SCRIPT_LINES if dag.keepalive else BASH_SCRIPT_LINES + script = "".join(f" {lit(line)},\n" for line in lines) return ( f"{doc}\n" "\n" @@ -1697,6 +1840,10 @@ def render_fluid_apply_dags( "render_fluid_apply_dags", "render_scheduled_dag", "rendered_dag_of", + "KEEPALIVE_APPLY_ARGS", + "KEEPALIVE_SCHEDULE", + "KEEPALIVE_SCRIPT_LINES", + "KEEPALIVE_TASK_TIMEOUT_S", "runs_only_outside_schedulers", "schedule_scope_for", "scheduled_builds", diff --git a/tests/cli/test_generate_schedule_build_graph.py b/tests/cli/test_generate_schedule_build_graph.py index 7da62bfa..bdb0081a 100644 --- a/tests/cli/test_generate_schedule_build_graph.py +++ b/tests/cli/test_generate_schedule_build_graph.py @@ -16,11 +16,14 @@ import logging import os from pathlib import Path +from types import SimpleNamespace import pytest import yaml from fluid_build.cli import generate_schedule +from fluid_build.cli.scaffold_composer import run as scaffold_composer +from fluid_build.schedulers.airflow import fluid_apply from tests.cli._schedule_dag_fixtures import StubAsset, load_dag, run_task from tests.schedulers.fluid_apply_fixtures import SPARK_CONTRACT, events_pipeline @@ -102,6 +105,30 @@ def test_a_contract_of_streams_alone_needs_no_dag( assert not (tmp_path / "out").exists() +def test_a_kept_alive_stream_gets_its_keepalive_dag( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + stream = dict(SPARK_CONTRACT["builds"][0], execution={"retries": {"maxAttempts": 5}}) + _project(tmp_path, dict(SPARK_CONTRACT, builds=[stream])) + assert _generate(tmp_path, monkeypatch) == 0 + assert sorted(p.name for p in (tmp_path / "out").iterdir()) == ["stream_raw_dag.py"] + + +@pytest.mark.parametrize("scheduler", ["prefect", "dagster"]) +def test_a_scheduler_with_no_keepalive_has_nothing_to_schedule_for_a_stream( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture, scheduler: str +) -> None: + """Only the Airflow DAGs keep a stream running: another scheduler writes nothing for it, + and is told where the keepalive DAG comes from.""" + stream = dict(SPARK_CONTRACT["builds"][0], execution={"retries": {"maxAttempts": 5}}) + _project(tmp_path, dict(SPARK_CONTRACT, builds=[stream])) + assert _generate(tmp_path, monkeypatch, "--scheduler", scheduler) == 0 + out = " ".join(capsys.readouterr().out.split()) + assert "No build needs a DAG" in out + assert "gets a keepalive DAG on Airflow: fluid generate schedule --scheduler airflow" in out + assert not (tmp_path / "out").exists() + + def test_dag_id_and_schedule_overrides_stay_refused_for_a_build_graph( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: @@ -114,10 +141,6 @@ def test_dag_id_and_schedule_overrides_stay_refused_for_a_build_graph( def test_the_composer_scaffold_agrees_that_streams_need_no_dag( tmp_path: Path, capsys: pytest.CaptureFixture ) -> None: - from types import SimpleNamespace - - from fluid_build.cli.scaffold_composer import run as scaffold_composer - contract = tmp_path / "contract.fluid.yaml" stream = dict(SPARK_CONTRACT["builds"][0], execution={"trigger": {"cron": "@daily"}}) contract.write_text(yaml.safe_dump(dict(SPARK_CONTRACT, builds=[stream])), "utf-8") @@ -128,6 +151,24 @@ def test_the_composer_scaffold_agrees_that_streams_need_no_dag( assert not (tmp_path / "dags").exists() +def test_the_composer_scaffold_leaves_a_kept_alive_stream_to_generate_schedule( + tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + """Its validate, plan and apply DAG cannot keep a stream running: it writes none, and says + which command writes the stream's keepalive DAG.""" + contract = tmp_path / "contract.fluid.yaml" + stream = dict(SPARK_CONTRACT["builds"][0], execution={"retries": {"maxAttempts": 5}}) + data = dict(SPARK_CONTRACT, builds=[stream]) + assert fluid_apply.has_scheduled_builds(data) # generate schedule writes its keepalive DAG + contract.write_text(yaml.safe_dump(data), "utf-8") + args = SimpleNamespace(contract=str(contract), env=None, out_dir=str(tmp_path / "dags")) + assert scaffold_composer(args, LOG) == 0 + out = " ".join(capsys.readouterr().out.split()) + assert "No DAG written." in out + assert "gets a keepalive DAG from fluid generate schedule" in out + assert not (tmp_path / "dags").exists() + + def test_regenerating_removes_a_dag_whose_build_now_runs_in_another( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture ) -> None: @@ -140,8 +181,6 @@ def test_regenerating_removes_a_dag_whose_build_now_runs_in_another( assert _generate(tmp_path, monkeypatch) == 0 out = tmp_path / "out" assert (out / "daily_users_dag.py").exists() - from fluid_build.schedulers.airflow import fluid_apply - rendered = fluid_apply.render_fluid_apply_dags(contract, env="aws", contract_path=CONTRACT) other_env = rendered["daily_users_dag.py"] (out / "daily_users_other_env_dag.py").write_text(other_env, encoding="utf-8") diff --git a/tests/schedulers/fluid_apply_fixtures.py b/tests/schedulers/fluid_apply_fixtures.py index 7868caec..61e9537a 100644 --- a/tests/schedulers/fluid_apply_fixtures.py +++ b/tests/schedulers/fluid_apply_fixtures.py @@ -261,8 +261,23 @@ def rules_and_any() -> Dict[str, Any]: return _with_builds(contract, contract["builds"][0], staging, mart, notify, report) -#: Contracts that use ``builds[].dependencies`` and dataset triggers. +def kept_alive_stream() -> Dict[str, Any]: + """The Spark contract whose stream declares execution.retries: a keepalive DAG beside the + daily batch, on a cadence the build's Airflow DAG config sets.""" + contract = copy.deepcopy(SPARK_CONTRACT) + contract["builds"][0]["execution"] = { + "retries": {"maxAttempts": 5}, + "orchestration": { + "engine": "airflow", + "airflow": {"dagId": "events_keepalive", "dagConfig": {"schedule": "*/10 * * * *"}}, + }, + } + return contract + + +#: Contracts that use ``builds[].dependencies``, dataset triggers or a kept-alive stream. GRAPH_CASES: Dict[str, Tuple[Dict[str, Any], Optional[str], str]] = { "events_pipeline": (events_pipeline(), None, "contracts/events/contract.fluid.yaml"), "rules_and_any": (rules_and_any(), "aws", "contract.fluid.yaml"), + "kept_alive_stream": (kept_alive_stream(), "aws", "native.fluid.yaml"), } diff --git a/tests/schedulers/golden/fluid_apply/kept_alive_stream/daily_curated_dag.py.txt b/tests/schedulers/golden/fluid_apply/kept_alive_stream/daily_curated_dag.py.txt new file mode 100644 index 00000000..cbdee237 --- /dev/null +++ b/tests/schedulers/golden/fluid_apply/kept_alive_stream/daily_curated_dag.py.txt @@ -0,0 +1,123 @@ +"""Scheduled fluid build, generated by fluid. Edit the contract, not this file. + +Every run executes this on the Airflow worker, with the constants below +filled in (`--env` is left out when FLUID_ENV_NAME is empty): + + cd "$FLUID_PROJECT_DIR" + fluid apply "$FLUID_PROJECT_DIR/$CONTRACT_PATH" --env "$FLUID_ENV_NAME" \\ + --mode amend-and-build --build-id "$BUILD_ID" --yes + +Worker requirements: + +- FLUID_PROJECT_DIR names the directory that holds the product checkout, + the directory CI ran the pipeline from. +- `fluid` is on PATH, or FLUID_BIN names the executable. +- The credentials the apply needs are in the worker environment. Only + these reach the fluid process: + + PATH HOME USER LOGNAME LANG LANGUAGE LC_ALL LC_CTYPE TZ TMPDIR + SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE HTTP_PROXY + HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy GLUE_ROLE_ARN + S3_STAGING_DIR S3_DATA_DIR GOOG_SERVICE_ACCOUNT_NAME + TESTCONTAINERS_HOST_OVERRIDE PYTHON_DOTENV_DISABLED + + every name matching + + FLUID_* AWS_* GOOGLE_* GCP_* GCLOUD_* CLOUDSDK_* AZURE_* ARM_* + SNOWFLAKE_* DATABRICKS_* PG* POSTGRES_* REDSHIFT_* ATHENA_* VAULT_* + BAO_* DBT_* DLT_* DATAHUB_* DMM_* ODCS_* ODPS_* TF_* OPENLINEAGE_* + OTEL_* + + the variables the contract reads (CONTRACT_ENV_NAMES), and any names + listed, space separated, in FLUID_DAG_ENV_PASSTHROUGH. fluid starts with + nothing else, and never with a variable whose name starts AIRFLOW. +""" + +from datetime import timedelta + +import pendulum + +try: # Airflow 3 + from airflow.sdk import DAG +except ImportError: # Airflow 2.x + from airflow import DAG + +try: # Airflow 3, or Airflow 2 with apache-airflow-providers-standard + from airflow.providers.standard.operators.bash import BashOperator +except ImportError: # Airflow 2.x core operator + from airflow.operators.bash import BashOperator + +try: # Airflow 3 + from airflow.sdk import Asset +except ImportError: # Airflow 2.x + from airflow.datasets import Dataset as Asset + +PRODUCT_ID = 'platform.events' +BUILD_ID = 'daily-curated' +CONTRACT_PATH = 'native.fluid.yaml' +FLUID_ENV_NAME = 'aws' +CONTRACT_ENV_NAMES = '' +SCHEDULE = '0 2 * * *' +TIMEZONE = 'UTC' +RETRIES = 3 + +BASH_COMMAND = "\n".join( + [ + 'set -euo pipefail', + ': "${FLUID_PROJECT_DIR:?set FLUID_PROJECT_DIR on the Airflow worker to the directory that holds the product checkout}"', + 'keep=" PATH HOME USER LOGNAME LANG LANGUAGE LC_ALL LC_CTYPE TZ TMPDIR SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE HTTP_PROXY HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy GLUE_ROLE_ARN S3_STAGING_DIR S3_DATA_DIR GOOG_SERVICE_ACCOUNT_NAME TESTCONTAINERS_HOST_OVERRIDE PYTHON_DOTENV_DISABLED ${FLUID_DAG_CONTRACT_ENV:-} ${FLUID_DAG_ENV_PASSTHROUGH:-} "', + 'pass=()', + 'for name in $(compgen -e); do', + ' case "$name" in', + ' AIRFLOW*) continue ;;', + ' FLUID_*|AWS_*|GOOGLE_*|GCP_*|GCLOUD_*|CLOUDSDK_*|AZURE_*|ARM_*|SNOWFLAKE_*|DATABRICKS_*|PG*|POSTGRES_*|REDSHIFT_*|ATHENA_*|VAULT_*|BAO_*|DBT_*|DLT_*|DATAHUB_*|DMM_*|ODCS_*|ODPS_*|TF_*|OPENLINEAGE_*|OTEL_*) ;;', + ' *) case "$keep" in *" $name "*) ;; *) continue ;; esac ;;', + ' esac', + ' pass+=("$name=${!name}")', + 'done', + 'cd -- "$FLUID_PROJECT_DIR"', + 'contract="$PWD/$FLUID_DAG_CONTRACT"', + 'if [ ! -f "$contract" ]; then', + ' echo "fluid: no contract at $contract (FLUID_PROJECT_DIR + $FLUID_DAG_CONTRACT)" >&2', + ' exit 2', + 'fi', + 'set -- apply "$contract"', + 'if [ -n "$FLUID_DAG_ENV" ]; then set -- "$@" --env "$FLUID_DAG_ENV"; fi', + 'set -- "$@" --mode amend-and-build --build-id "$FLUID_DAG_BUILD_ID" --yes', + 'bin="${FLUID_BIN:-fluid}"', + 'case "$bin" in *=*) echo "fluid: FLUID_BIN may not contain =" >&2; exit 2 ;; esac', + 'exec env -i ${pass[@]+"${pass[@]}"} "$bin" "$@"', + ] +) + +with DAG( + dag_id='platform.events__aws__daily-curated', + description='fluid apply platform.events --build-id daily-curated', + schedule=SCHEDULE, + start_date=pendulum.datetime(2026, 1, 1, tz=TIMEZONE), + catchup=False, + max_active_runs=1, + default_args={ + "owner": "fluid", + "retries": RETRIES, + "retry_delay": timedelta(minutes=5), + "execution_timeout": timedelta(seconds=32400), + }, + tags=["fluid", PRODUCT_ID[:100]], + doc_md=__doc__, +) as dag: + BashOperator( + task_id="fluid_apply", + bash_command=BASH_COMMAND, + env={ + "FLUID_DAG_CONTRACT": CONTRACT_PATH, + "FLUID_DAG_ENV": FLUID_ENV_NAME, + "FLUID_DAG_BUILD_ID": BUILD_ID, + "FLUID_DAG_CONTRACT_ENV": CONTRACT_ENV_NAMES, + }, + append_env=True, + skip_on_exit_code=None, + outlets=[ + Asset('s3://lake/iceberg/curated/events_daily'), + ], + ) diff --git a/tests/schedulers/golden/fluid_apply/kept_alive_stream/stream_raw_dag.py.txt b/tests/schedulers/golden/fluid_apply/kept_alive_stream/stream_raw_dag.py.txt new file mode 100644 index 00000000..ff0cd8c2 --- /dev/null +++ b/tests/schedulers/golden/fluid_apply/kept_alive_stream/stream_raw_dag.py.txt @@ -0,0 +1,126 @@ +"""Scheduled fluid build, generated by fluid. Edit the contract, not this file. + +Every run reconciles one always-on stream on the Airflow worker, with the +constants below filled in (`--env` is left out when FLUID_ENV_NAME is empty): + + cd "$FLUID_PROJECT_DIR" + fluid apply "$FLUID_PROJECT_DIR/$CONTRACT_PATH" --env "$FLUID_ENV_NAME" \\ + --mode reconcile --build-id "$BUILD_ID" --yes + +It runs no tofu apply. When the stream died (a Spot interruption, a failed +driver), it starts again on its checkpoint, within the restart budget the +build's execution.retries sets, so the task itself never retries. The run +fails when the stream cannot be kept running: it drifted from the contract, +nothing is deployed, the job config the last apply deployed cannot be read, +its cluster ended, or the restarts are spent. It can also fail when this +worker's FLUID_SPARK_* settings or forge release differ from the CI apply's. +A run cancelled on purpose stays stopped until fluid apply starts it. It +runs on SCHEDULE, one run at a time, with no catch-up. The build's +execution.orchestration.airflow.dagConfig.schedule sets SCHEDULE. + +Worker requirements: + +- FLUID_PROJECT_DIR names the directory that holds the product checkout, + the directory CI ran the pipeline from. +- `fluid` is on PATH, or FLUID_BIN names the executable. +- The credentials the apply needs are in the worker environment. Only + these reach the fluid process: + + PATH HOME USER LOGNAME LANG LANGUAGE LC_ALL LC_CTYPE TZ TMPDIR + SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE HTTP_PROXY + HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy GLUE_ROLE_ARN + S3_STAGING_DIR S3_DATA_DIR GOOG_SERVICE_ACCOUNT_NAME + TESTCONTAINERS_HOST_OVERRIDE PYTHON_DOTENV_DISABLED + + every name matching + + FLUID_* AWS_* GOOGLE_* GCP_* GCLOUD_* CLOUDSDK_* AZURE_* ARM_* + SNOWFLAKE_* DATABRICKS_* PG* POSTGRES_* REDSHIFT_* ATHENA_* VAULT_* + BAO_* DBT_* DLT_* DATAHUB_* DMM_* ODCS_* ODPS_* TF_* OPENLINEAGE_* + OTEL_* + + the variables the contract reads (CONTRACT_ENV_NAMES), and any names + listed, space separated, in FLUID_DAG_ENV_PASSTHROUGH. fluid starts with + nothing else, and never with a variable whose name starts AIRFLOW. +""" + +from datetime import timedelta + +import pendulum + +try: # Airflow 3 + from airflow.sdk import DAG +except ImportError: # Airflow 2.x + from airflow import DAG + +try: # Airflow 3, or Airflow 2 with apache-airflow-providers-standard + from airflow.providers.standard.operators.bash import BashOperator +except ImportError: # Airflow 2.x core operator + from airflow.operators.bash import BashOperator + +PRODUCT_ID = 'platform.events' +BUILD_ID = 'stream-raw' +CONTRACT_PATH = 'native.fluid.yaml' +FLUID_ENV_NAME = 'aws' +CONTRACT_ENV_NAMES = '' +SCHEDULE = '*/10 * * * *' +TIMEZONE = 'UTC' +RETRIES = 0 + +BASH_COMMAND = "\n".join( + [ + 'set -euo pipefail', + ': "${FLUID_PROJECT_DIR:?set FLUID_PROJECT_DIR on the Airflow worker to the directory that holds the product checkout}"', + 'keep=" PATH HOME USER LOGNAME LANG LANGUAGE LC_ALL LC_CTYPE TZ TMPDIR SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE HTTP_PROXY HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy GLUE_ROLE_ARN S3_STAGING_DIR S3_DATA_DIR GOOG_SERVICE_ACCOUNT_NAME TESTCONTAINERS_HOST_OVERRIDE PYTHON_DOTENV_DISABLED ${FLUID_DAG_CONTRACT_ENV:-} ${FLUID_DAG_ENV_PASSTHROUGH:-} "', + 'pass=()', + 'for name in $(compgen -e); do', + ' case "$name" in', + ' AIRFLOW*) continue ;;', + ' FLUID_*|AWS_*|GOOGLE_*|GCP_*|GCLOUD_*|CLOUDSDK_*|AZURE_*|ARM_*|SNOWFLAKE_*|DATABRICKS_*|PG*|POSTGRES_*|REDSHIFT_*|ATHENA_*|VAULT_*|BAO_*|DBT_*|DLT_*|DATAHUB_*|DMM_*|ODCS_*|ODPS_*|TF_*|OPENLINEAGE_*|OTEL_*) ;;', + ' *) case "$keep" in *" $name "*) ;; *) continue ;; esac ;;', + ' esac', + ' pass+=("$name=${!name}")', + 'done', + 'cd -- "$FLUID_PROJECT_DIR"', + 'contract="$PWD/$FLUID_DAG_CONTRACT"', + 'if [ ! -f "$contract" ]; then', + ' echo "fluid: no contract at $contract (FLUID_PROJECT_DIR + $FLUID_DAG_CONTRACT)" >&2', + ' exit 2', + 'fi', + 'set -- apply "$contract"', + 'if [ -n "$FLUID_DAG_ENV" ]; then set -- "$@" --env "$FLUID_DAG_ENV"; fi', + 'set -- "$@" --mode reconcile --build-id "$FLUID_DAG_BUILD_ID" --yes', + 'bin="${FLUID_BIN:-fluid}"', + 'case "$bin" in *=*) echo "fluid: FLUID_BIN may not contain =" >&2; exit 2 ;; esac', + 'exec env -i ${pass[@]+"${pass[@]}"} "$bin" "$@"', + ] +) + +with DAG( + dag_id='platform.events__aws__stream-raw', + description='keep fluid stream platform.events stream-raw running', + schedule=SCHEDULE, + start_date=pendulum.datetime(2026, 1, 1, tz=TIMEZONE), + catchup=False, + max_active_runs=1, + default_args={ + "owner": "fluid", + "retries": RETRIES, + "retry_delay": timedelta(minutes=5), + "execution_timeout": timedelta(seconds=1800), + }, + tags=["fluid", PRODUCT_ID[:100]], + doc_md=__doc__, +) as dag: + BashOperator( + task_id="fluid_keepalive", + bash_command=BASH_COMMAND, + env={ + "FLUID_DAG_CONTRACT": CONTRACT_PATH, + "FLUID_DAG_ENV": FLUID_ENV_NAME, + "FLUID_DAG_BUILD_ID": BUILD_ID, + "FLUID_DAG_CONTRACT_ENV": CONTRACT_ENV_NAMES, + }, + append_env=True, + skip_on_exit_code=None, + ) diff --git a/tests/schedulers/test_fluid_apply_airflow_live.py b/tests/schedulers/test_fluid_apply_airflow_live.py index d7493c9d..703a467d 100644 --- a/tests/schedulers/test_fluid_apply_airflow_live.py +++ b/tests/schedulers/test_fluid_apply_airflow_live.py @@ -48,6 +48,7 @@ "edges": sorted([t.task_id, d] for t in dag.tasks for d in t.downstream_task_ids), "rules": {t.task_id: getattr(t.trigger_rule, "value", t.trigger_rule) for t in dag.tasks}, "outlets": {t.task_id: [o.uri for o in t.outlets] for t in dag.tasks}, + "retries": {t.task_id: t.retries for t in dag.tasks}, "timetable": type(dag.timetable).__name__, } print(json.dumps(out)) @@ -63,7 +64,8 @@ def _expected(dag: fluid_apply.ScheduledDag, major: str) -> Dict[str, Any]: only = len(dag.tasks) == 1 - task_id = {t.build_id: ("fluid_apply" if only else t.build_id) for t in dag.tasks} + single = "fluid_keepalive" if dag.keepalive else "fluid_apply" + task_id = {t.build_id: (single if only else t.build_id) for t in dag.tasks} if dag.schedule.datasets and dag.schedule.cron is not None: timetable = _OR_TIME[major] else: @@ -73,6 +75,7 @@ def _expected(dag: fluid_apply.ScheduledDag, major: str) -> Dict[str, Any]: "edges": sorted([task_id[u], task_id[t.build_id]] for t in dag.tasks for u in t.upstream), "rules": {task_id[t.build_id]: t.trigger_rule or "all_success" for t in dag.tasks}, "outlets": {task_id[t.build_id]: list(t.outlets) for t in dag.tasks}, + "retries": {task_id[t.build_id]: t.retries for t in dag.tasks}, "timetable": timetable, } diff --git a/tests/schedulers/test_fluid_apply_keepalive.py b/tests/schedulers/test_fluid_apply_keepalive.py new file mode 100644 index 00000000..f8b69d86 --- /dev/null +++ b/tests/schedulers/test_fluid_apply_keepalive.py @@ -0,0 +1,192 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""An always-on Spark stream that declares ``execution.retries`` gets a keepalive DAG. + +Its one task runs ``fluid apply --mode reconcile --build-id `` +(:data:`fluid_apply.KEEPALIVE_APPLY_ARGS`) every few minutes, so a stream a Spot interruption +stopped starts again. One run at a time, no catch-up, no Airflow retries: the restart budget is +the build's own ``execution.retries``. +""" + +from __future__ import annotations + +import copy +from datetime import timedelta +from pathlib import Path +from typing import Any, Dict + +import pytest + +from fluid_build.schedulers.airflow import fluid_apply +from tests.cli._schedule_dag_fixtures import load_dag, run_task +from tests.schedulers.fluid_apply_fixtures import SPARK_CONTRACT + + +def _contract(**execution: Any) -> Dict[str, Any]: + contract = copy.deepcopy(SPARK_CONTRACT) + contract["builds"][0]["execution"] = {"retries": {"maxAttempts": 5}, **execution} + return contract + + +def _files(contract: Dict[str, Any], env: Any = "aws") -> Dict[str, str]: + return fluid_apply.render_fluid_apply_dags(contract, env=env, contract_path="native.fluid.yaml") + + +def _keepalive_config(schedule: Any) -> Dict[str, Any]: + return { + "orchestration": { + "engine": "airflow", + "airflow": {"dagId": "events_keepalive", "dagConfig": {"schedule": schedule}}, + } + } + + +@pytest.mark.parametrize("airflow3", [True, False]) +def test_a_stream_with_retries_gets_a_keepalive_dag(monkeypatch, airflow3: bool): + files = _files(_contract()) + assert sorted(files) == ["daily_curated_dag.py", "stream_raw_dag.py"] + loaded = load_dag(files["stream_raw_dag.py"], monkeypatch, airflow3=airflow3) + assert loaded.dag["dag_id"] == "platform.events__aws__stream-raw" + assert loaded.dag["schedule"] == fluid_apply.KEEPALIVE_SCHEDULE + assert loaded.dag["max_active_runs"] == 1 and loaded.dag["catchup"] is False + assert loaded.dag["default_args"]["retries"] == 0 + assert loaded.dag["default_args"]["execution_timeout"] == timedelta( + seconds=fluid_apply.KEEPALIVE_TASK_TIMEOUT_S + ) + (task,) = loaded.tasks + assert task["task_id"] == "fluid_keepalive" + assert task["env"]["FLUID_DAG_BUILD_ID"] == "stream-raw" + assert "outlets" not in task + assert task["bash_command"] == "\n".join(fluid_apply.KEEPALIVE_SCRIPT_LINES) + + +def test_each_run_reconciles_that_one_stream(tmp_path: Path, monkeypatch): + project = tmp_path / "project" + project.mkdir() + (project / "native.fluid.yaml").write_text("id: x\n", encoding="utf-8") + (task,) = load_dag(_files(_contract())["stream_raw_dag.py"], monkeypatch).tasks + run = run_task(task, tmp_path, {"FLUID_PROJECT_DIR": str(project)}) + assert run.returncode == 0, run.stderr + assert run.argv is not None + assert run.argv[2:] == [ + "--env", + "aws", + *fluid_apply.KEEPALIVE_APPLY_ARGS, + "--build-id", + "stream-raw", + "--yes", + ] + + +def test_the_keepalive_runs_apply_in_reconcile_mode(): + assert fluid_apply.KEEPALIVE_APPLY_ARGS == ("--mode", "reconcile") + tail = [line for line in fluid_apply.KEEPALIVE_SCRIPT_LINES if "--build-id" in line] + assert tail == ['set -- "$@" --mode reconcile --build-id "$FLUID_DAG_BUILD_ID" --yes'] + # The keepalive script is the apply script but for that line, and holds no contract text. + differ = set(fluid_apply.KEEPALIVE_SCRIPT_LINES) ^ set(fluid_apply.BASH_SCRIPT_LINES) + assert len(differ) == 2 + for word in fluid_apply.KEEPALIVE_APPLY_ARGS: + assert word.replace("-", "").isalnum(), word + + +def test_apply_arguments_render_one_space_apart_even_when_there_are_none(): + assert fluid_apply._apply_args((), '"$B"') == '--build-id "$B" --yes' + assert fluid_apply._apply_args(("--mode", "x"), '"$B"') == '--mode x --build-id "$B" --yes' + source = _files(_contract())["stream_raw_dag.py"] + assert " --build-id" not in source and "reconcile " not in source + + +def test_fluid_apply_accepts_the_keepalive_argv(): + """The DAG never runs an apply the CLI would refuse: the mode exists, and it takes --build-id.""" + from fluid_build.cli import build_parser + from fluid_build.cli.apply import _check_build_id_mode + from fluid_build.forge.core.apply_modes import parse_mode + + argv = ["apply", "native.fluid.yaml", "--env", "aws", *fluid_apply.KEEPALIVE_APPLY_ARGS] + args = build_parser().parse_args([*argv, "--build-id", "stream-raw", "--yes"]) + assert (args.build_id, args.yes) == ("stream-raw", True) + mode = parse_mode(args.mode) + assert mode.value == "reconcile" + _check_build_id_mode(args.build_id, mode) + + +def test_a_dataset_trigger_on_what_only_a_kept_alive_stream_writes_is_refused(): + """A keepalive DAG declares no outlets, so a trigger on an expose only the stream writes + would never fire: it is refused, as it is for a stream with no keepalive.""" + contract = _contract() + trigger = { + "type": "dataset", + "datasets": [{"productId": contract["id"], "exposeId": "events_raw"}], + } + contract["builds"].append( + {"id": "consumer", "engine": "dbt", "outputs": [], "execution": {"trigger": trigger}} + ) + with pytest.raises(fluid_apply.ScheduleRenderError) as exc: + _files(contract) + message = str(exc.value) + assert "is written by 'stream-raw', whose keepalive DAG declares no outlets" in message + assert "the trigger would never fire" in message + + +def test_a_stream_its_platform_keeps_running_gets_no_keepalive(): + """trigger.type: continuous puts a build outside every scheduler, retries or not.""" + assert sorted(_files(_contract(trigger={"type": "continuous"}))) == ["daily_curated_dag.py"] + + +def test_without_retries_a_stream_still_gets_no_dag(): + assert sorted(_files(copy.deepcopy(SPARK_CONTRACT))) == ["daily_curated_dag.py"] + + +def test_dag_config_schedule_sets_the_cadence(monkeypatch): + source = _files(_contract(**_keepalive_config("*/2 * * * *")))["stream_raw_dag.py"] + assert load_dag(source, monkeypatch).dag["schedule"] == "*/2 * * * *" + + +@pytest.mark.parametrize( + "schedule, message", + [ + ("@once", "a keepalive DAG reconciles the stream on a recurring cron"), + ("*/90 * * *", "is not a five-field cron expression"), + ], +) +def test_a_cadence_airflow_cannot_run_is_refused(schedule: str, message: str): + with pytest.raises(fluid_apply.ScheduleRenderError) as exc: + _files(_contract(**_keepalive_config(schedule))) + assert "execution.orchestration.airflow.dagConfig.schedule" in str(exc.value) + assert message in str(exc.value) + + +@pytest.mark.parametrize("engine", ["none", "dagster"]) +def test_a_stream_on_another_engine_gets_no_keepalive(engine: str): + contract = _contract(orchestration={"engine": engine}) + assert sorted(_files(contract)) == ["daily_curated_dag.py"] + + +def test_a_stream_that_runs_to_completion_is_a_batch_not_a_keepalive(): + contract = _contract() + contract["builds"][0]["properties"]["stream"]["trigger"] = {"availableNow": True} + contract["builds"][0]["execution"]["trigger"] = {"type": "schedule", "schedule": "@hourly"} + source = _files(contract)["stream_raw_dag.py"] + assert 'task_id="fluid_apply"' in source and "fluid_keepalive" not in source + + +def test_a_streams_only_contract_with_retries_needs_a_dag(): + contract = _contract() + contract["builds"] = contract["builds"][:1] + assert fluid_apply.has_scheduled_builds(contract) + assert not fluid_apply.runs_only_outside_schedulers(contract) + assert sorted(_files(contract)) == ["stream_raw_dag.py"] + + +def test_a_batch_still_does_not_wait_for_the_stream_it_depends_on(): + contract = _contract() + contract["builds"][1]["dependencies"] = [{"buildId": "stream-raw"}] + (daily,) = [d for d in fluid_apply.scheduled_dags(contract) if not d.keepalive] + assert [t.build_id for t in daily.tasks] == ["daily-curated"] + assert daily.unwaited[0][0] == "stream-raw" + + +def test_the_keepalive_reads_back_as_the_products_dag(tmp_path: Path): + (tmp_path / "stream_raw_dag.py").write_text(_files(_contract())["stream_raw_dag.py"]) + assert fluid_apply.rendered_dag_of(tmp_path / "stream_raw_dag.py") == ("platform.events", "aws") diff --git a/tests/test_scheduler_framework.py b/tests/test_scheduler_framework.py index 2e7320fd..aec19107 100644 --- a/tests/test_scheduler_framework.py +++ b/tests/test_scheduler_framework.py @@ -28,11 +28,14 @@ @pytest.fixture(autouse=True) -def _clean_registry(): - """Reset the scheduler registry before and after each test.""" - _reset_registry() - yield - _reset_registry() +def _clean_registry(monkeypatch: pytest.MonkeyPatch) -> None: + """Give each test an empty scheduler registry, and put the shared one back after it. + + The built-in schedulers register once, when ``fluid_build.schedulers`` is first + imported. Emptying the shared registry would hide them from every later test in the + same process, and the test order is random. + """ + monkeypatch.setattr("fluid_build.schedulers.registry._SCHEDULERS", {}) class _DummyScheduler(ScheduleEngine):