From 50b61daabf7e5d0a628a69c346064176c427eb94 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Wed, 18 Nov 2026 18:34:08 +0100 Subject: [PATCH 01/10] feat(schedule): keep an always-on stream running with a keepalive DAG A forge Spark stream that is not trigger.availableNow and declares execution.retries gets a DAG of its own: one task that runs the reconcile-only fluid apply for that stream (KEEPALIVE_APPLY_ARGS, the one place it is named), every 5 minutes or on the build's execution.orchestration.airflow.dagConfig.schedule, with max_active_runs=1, no catch-up and no Airflow retries, so the build's own retries bound the restarts. It declares no outlets: a stream updates its tables all the time. A contract whose only builds are such streams now needs a DAG. --- AGENTS.md | 2 +- CHANGELOG.md | 9 ++ fluid_build/schedulers/airflow/fluid_apply.py | 127 ++++++++++++++- tests/schedulers/fluid_apply_fixtures.py | 17 +- .../daily_curated_dag.py.txt | 123 ++++++++++++++ .../kept_alive_stream/stream_raw_dag.py.txt | 121 ++++++++++++++ .../test_fluid_apply_airflow_live.py | 5 +- .../schedulers/test_fluid_apply_keepalive.py | 150 ++++++++++++++++++ 8 files changed, 545 insertions(+), 9 deletions(-) create mode 100644 tests/schedulers/golden/fluid_apply/kept_alive_stream/daily_curated_dag.py.txt create mode 100644 tests/schedulers/golden/fluid_apply/kept_alive_stream/stream_raw_dag.py.txt create mode 100644 tests/schedulers/test_fluid_apply_keepalive.py diff --git a/AGENTS.md b/AGENTS.md index 2bb6f437..74fba4c2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -154,7 +154,7 @@ forge-cli/ | Provider plugins (planning) | `fluid_build/providers/{local,gcp,aws,snowflake,odps,odcs}/` — `plan()` emits abstract actions | | **IaC emitter (apply)** | `fluid_build/iac/` — modular OpenTofu plugin registry. `cli/_apply_opentofu_engine.py` compiles the contract to `main.tf.json` and shells `tofu` for cloud providers. `cli/generate_iac.py` is `fluid generate iac` for review-only emit. One plugin per cloud at `iac/providers/{aws,gcp,snowflake}.py` (dbt-adapter pattern). | | Path-B schedulers | `fluid_build/providers/aws/plan/schedule.py` (EventBridge/MWAA/Lambda/Step-Functions via stage 6 `SchedulePlanner`) | -| Airflow DAGs | `fluid_build/schedulers/airflow/` is the one renderer. `render_dag()` (`__init__.py`, operators in `_operators.py`) renders a contract's `orchestration` section for gcp/aws/snowflake; `fluid_apply.py` renders each build as one `fluid apply --mode amend-and-build --build-id` task, a forge Spark build included (its build phase starts the run and waits; a stream that is not `availableNow` gets no DAG). Builds that `builds[].dependencies` link share one DAG (`scheduled_dags()`: the graph and the schedule; `render_build_task()`: the one function that writes a build's node), on the one trigger they declare: a cron, `trigger.datasets` (`dataset`), or both (`schedule_and_dataset`). Every task declares its build's exposes as Airflow asset `outlets`, named as Airflow's providers name the storage (`_assets.py`). `fluid generate schedule`, `fluid export --engine airflow` and `fluid generate-airflow` (a deprecated alias of the first) all come through it. Task variables go through `codegen_utils.unique_task_identifiers`. | +| Airflow DAGs | `fluid_build/schedulers/airflow/` is the one renderer. `render_dag()` (`__init__.py`, operators in `_operators.py`) renders a contract's `orchestration` section for gcp/aws/snowflake; `fluid_apply.py` renders each build as one `fluid apply --mode amend-and-build --build-id` task, a forge Spark build included (its build phase starts the run and waits; a stream that is not `availableNow` gets no DAG). Builds that `builds[].dependencies` link share one DAG (`scheduled_dags()`: the graph and the schedule; `render_build_task()`: the one function that writes a build's node), on the one trigger they declare: a cron, `trigger.datasets` (`dataset`), or both (`schedule_and_dataset`). Every task declares its build's exposes as Airflow asset `outlets`, named as Airflow's providers name the storage (`_assets.py`). A Spark stream that declares `execution.retries` gets a keepalive DAG whose task runs the reconcile-only apply (`KEEPALIVE_APPLY_ARGS`). `fluid generate schedule`, `fluid export --engine airflow` and `fluid generate-airflow` (a deprecated alias of the first) all come through it. Task variables go through `codegen_utils.unique_task_identifiers`. | | Contract schemas | `fluid_build/schemas/*.json` (0.7.2 is latest; `SchemaManager.latest_bundled_version()` is the dynamic lookup) | | Policy engine | `fluid_build/policy/` (compiler, agent_policy, sovereignty, guardrails) | | Plan-binding helpers | `fluid_build/forge/core/plan_digest.py` — `inject_digests`, `verify_plan_binding`, `PlanBindingError` | diff --git a/CHANGELOG.md b/CHANGELOG.md index d04f9a1c..08f74e19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -116,6 +116,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 with the type and mode apply wrote, and `fluid diff` reads a map as it reads an array and every spelling of a struct (`struct`, `record`, `row`) as the RECORD BigQuery reports. +- **A keepalive DAG for an always-on stream that declares `execution.retries`.** `fluid + generate schedule` gives each forge Spark stream that is not `trigger.availableNow` and + declares `execution.retries` a small DAG whose one task runs the reconcile-only `fluid apply` + for that stream, so a stream a Spot interruption or a failed driver stopped starts again. It + runs every 5 minutes, or on the build's `execution.orchestration.airflow.dagConfig.schedule`, + one run at a time, with no catch-up and no Airflow retries: the build's own retries bound the + restarts. The task declares no outlets. A contract whose only builds are such streams now gets + this DAG instead of "No build needs a DAG". + - **`fluid generate schedule` renders the build graph: `builds[].dependencies` and dataset triggers.** Builds that `builds[].dependencies` link now run in one Airflow DAG, one `fluid apply --build-id` task per build, each after the builds it names. A dependency's diff --git a/fluid_build/schedulers/airflow/fluid_apply.py b/fluid_build/schedulers/airflow/fluid_apply.py index 725017b5..c2721735 100644 --- a/fluid_build/schedulers/airflow/fluid_apply.py +++ b/fluid_build/schedulers/airflow/fluid_apply.py @@ -124,6 +124,19 @@ #: destructive mode. APPLY_MODE = "amend-and-build" +#: What a keepalive DAG passes to ``fluid apply`` after the contract and ``--env``, before +#: ``--build-id --yes``: reconcile that one stream, restarting it when it stopped, and +#: change nothing else. The one place the reconcile-only apply is named. PLACEHOLDER: the +#: reconcile-only apply path does not exist yet; set the real arguments here once it does. +KEEPALIVE_APPLY_ARGS: Tuple[str, ...] = ("--mode", "reconcile") + +#: How often a keepalive DAG reconciles its stream, unless the build's +#: ``execution.orchestration.airflow.dagConfig.schedule`` names another cron. +KEEPALIVE_SCHEDULE = "*/5 * * * *" + +#: How long one keepalive run may take before Airflow stops it; the next run tries again. +KEEPALIVE_TASK_TIMEOUT_S = 30 * 60 + #: Contract path baked into the DAG when the caller cannot know the real one #: (a bundle carries no source path). Matches fluid's own discovery default #: and the generated CI pipelines' ``CONTRACT`` default. @@ -337,6 +350,8 @@ class ScheduledDag: tasks: Tuple[BuildTask, ...] #: ``(build id, why)``: builds a task depends on that no DAG runs, so none waits for them. unwaited: Tuple[Tuple[str, str], ...] = () + #: A keepalive DAG: its one task reconciles an always-on stream (:data:`KEEPALIVE_APPLY_ARGS`). + keepalive: bool = False # Where a build stands in the schedule (:func:`_classify`). @@ -434,6 +449,40 @@ def _outside_why(build: Dict[str, Any], managed: bool) -> str: return "a job that runs on its own platform (a Flink job, for example)" +def _keeps_alive(build: Dict[str, Any], managed: bool) -> bool: + """Whether a keepalive DAG watches the build: a forge Spark stream (one that is not + ``trigger.availableNow``) that declares ``execution.retries``, on Airflow's engine.""" + if _spark_kind(build, managed) != "streaming": + return False + execution = build.get("execution") + if not isinstance(execution, dict) or not isinstance(execution.get("retries"), dict): + return False + own = execution.get("orchestration") + return not (isinstance(own, dict) and own.get("engine") not in (None, "", DEFAULT_ENGINE)) + + +def _keepalive_schedule(build: Dict[str, Any]) -> str: + """The cron a keepalive DAG runs on: ``execution.orchestration.airflow.dagConfig.schedule``, + else :data:`KEEPALIVE_SCHEDULE`.""" + own = (build.get("execution") or {}).get("orchestration") + airflow = own.get("airflow") if isinstance(own, dict) else None + config = airflow.get("dagConfig") if isinstance(airflow, dict) else None + value = config.get("schedule") if isinstance(config, dict) else None + if value in (None, ""): + return KEEPALIVE_SCHEDULE + where = f"{build.get('id')!r}: execution.orchestration.airflow.dagConfig.schedule" + try: + cron = _validate_schedule(value, str(build.get("id"))) + except ScheduleRenderError as exc: + raise ScheduleRenderError(f"{where}: {exc}") from None + if cron == "@once": + raise ScheduleRenderError( + f"{where}: a keepalive DAG reconciles the stream on a recurring cron, and @once " + "is not one" + ) + return cron + + def _trigger_schedule(build: Dict[str, Any], managed: bool = True) -> Optional[Dict[str, Any]]: """The build's cron trigger, or ``None`` when it has none Airflow should run.""" kind, trigger = _classify(build, managed) @@ -447,7 +496,11 @@ def has_scheduled_builds(contract: Dict[str, Any]) -> bool: if not isinstance(builds, list): return False managed = _managed_lane(contract) - return any(isinstance(b, dict) and _classify(b, managed)[0] in _DAG_TRIGGERS for b in builds) + return any( + isinstance(b, dict) + and (_classify(b, managed)[0] in _DAG_TRIGGERS or _keeps_alive(b, managed)) + for b in builds + ) def runs_only_outside_schedulers(contract: Dict[str, Any]) -> bool: @@ -457,7 +510,9 @@ def runs_only_outside_schedulers(contract: Dict[str, Any]) -> bool: if not builds: return False managed = _managed_lane(contract) - return all(_classify(b, managed)[0] == _OUTSIDE for b in builds) + return all( + _classify(b, managed)[0] == _OUTSIDE and not _keeps_alive(b, managed) for b in builds + ) def has_explicit_tasks(contract: Dict[str, Any]) -> bool: @@ -987,6 +1042,30 @@ def check_expose_refs(self, dags: List[ScheduledDag]) -> None: "schedule (trigger.type: schedule)" ) + def keepalives(self) -> List[ScheduledDag]: + """One keepalive DAG per always-on stream that declares ``execution.retries``. + + Its one task reconciles the stream every few minutes, so a stream a Spot interruption + or a failed driver stopped starts again; the build's own retries bound how often. It + has no outlets: a stream updates its tables all the time, and an asset event on every + run would start each DAG scheduled on them. + """ + out: List[ScheduledDag] = [] + for build in self.builds: + if not _keeps_alive(build, self.managed): + continue + build_id = _task_build_id(build) + _check_spark_ids(self.contract, build_id) + out.append( + ScheduledDag( + build_id=build_id, + schedule=DagSchedule(cron=_keepalive_schedule(build), timezone="UTC"), + tasks=(BuildTask(build_id, 0, KEEPALIVE_TASK_TIMEOUT_S),), + keepalive=True, + ) + ) + return out + def _why_no_task(self, build: Dict[str, Any]) -> str: kind, trigger = _classify(build, self.managed) if kind == _OUTSIDE: @@ -1055,6 +1134,7 @@ def scheduled_dags(contract: Dict[str, Any]) -> List[ScheduledDag]: return [] graph = _BuildGraph(contract, [b for b in raw_builds if isinstance(b, dict)]) dags = [dag for members in graph.groups() if (dag := graph.dag(members)) is not None] + dags += graph.keepalives() seen: Set[str] = set() for task in (t for dag in dags for t in dag.tasks): if task.build_id in seen: @@ -1278,6 +1358,15 @@ def rendered_dag_of(path: Path) -> Optional[Tuple[str, str]]: BASH_SCRIPT_LINES = _SCRIPT_HEAD + _APPLY_TAIL + _SCRIPT_END +_KEEPALIVE_TAIL = ( + 'set -- apply "$contract"', + 'if [ -n "$FLUID_DAG_ENV" ]; then set -- "$@" --env "$FLUID_DAG_ENV"; fi', + 'set -- "$@" ' + " ".join(KEEPALIVE_APPLY_ARGS) + ' --build-id "$FLUID_DAG_BUILD_ID" --yes', +) + +#: The script of a keepalive DAG's task: :data:`BASH_SCRIPT_LINES` but for the apply arguments. +KEEPALIVE_SCRIPT_LINES = _SCRIPT_HEAD + _KEEPALIVE_TAIL + _SCRIPT_END + def _wrap(words: List[str], indent: str, width: int = 76) -> str: lines: List[str] = [] @@ -1317,6 +1406,21 @@ def _wrap(words: List[str], indent: str, width: int = 76) -> str: ' --mode amend-and-build --build-id "$BUILD_ID" --yes\n' ) +_KEEPALIVE_RUN = ( + "Every run reconciles one always-on stream on the Airflow worker, with the\n" + "constants below filled in (`--env` is left out when FLUID_ENV_NAME is empty):\n" + "\n" + ' cd "$FLUID_PROJECT_DIR"\n' + ' fluid apply "$FLUID_PROJECT_DIR/$CONTRACT_PATH" --env "$FLUID_ENV_NAME" \\\\\n' + f' {" ".join(KEEPALIVE_APPLY_ARGS)} --build-id "$BUILD_ID" --yes\n' + "\n" + "It starts the stream again when it stopped (a Spot interruption, a failed\n" + "driver) and changes nothing else; the build's execution.retries bound the\n" + "restarts, so the task itself never retries. It runs on SCHEDULE, one run\n" + "at a time, with no catch-up. The build's\n" + "execution.orchestration.airflow.dagConfig.schedule sets SCHEDULE.\n" +) + _BUILDS_RUN = ( "Each task runs one build on the Airflow worker, with the constants below\n" "filled in (`--env` is left out when FLUID_ENV_NAME is empty) and the\n" @@ -1447,7 +1551,7 @@ def _schedule_expr(schedule: DagSchedule) -> str: ) -def render_build_task(task: BuildTask, *, only: bool) -> str: +def render_build_task(task: BuildTask, *, only: bool, keepalive: bool = False) -> str: """The source of one build's node: a BashOperator that runs ``fluid apply --build-id``. This is the one function that decides what runs a build; the DAG's schedule and its edges @@ -1458,7 +1562,8 @@ def render_build_task(task: BuildTask, *, only: bool) -> str: """ lit = py_str_literal if only: - head, task_id, build_id = " BashOperator(\n", '"fluid_apply"', "BUILD_ID" + name = '"fluid_keepalive"' if keepalive else '"fluid_apply"' + head, task_id, build_id = " BashOperator(\n", name, "BUILD_ID" else: head = f" tasks[{lit(task.build_id)}] = BashOperator(\n" task_id = build_id = lit(task.build_id) @@ -1523,6 +1628,11 @@ def _edges(tasks: Tuple[BuildTask, ...]) -> str: def _dag_doc(product_id: str, dag: ScheduledDag) -> Tuple[str, str]: """The DAG file's docstring and the DAG's ``description``.""" + if dag.keepalive: + return ( + _doc("Scheduled fluid build", _KEEPALIVE_RUN, []), + f"keep fluid stream {product_id} {dag.build_id} running", + ) only = len(dag.tasks) == 1 notes = [] if only else [_TRIGGER_RULES_NOTE] notes += [n for n in (_schedule_note(dag.schedule), _unwaited_note(dag.unwaited)) if n] @@ -1560,7 +1670,7 @@ def _dag_tasks(dag: ScheduledDag) -> Tuple[str, str]: ' "retry_delay": timedelta(minutes=5),\n' f' "execution_timeout": {_timeout_expr(task.task_timeout_s)},\n' ) - return default_args, render_build_task(task, only=True) + return default_args, render_build_task(task, only=True, keepalive=dag.keepalive) nodes = "".join(render_build_task(t, only=False) for t in dag.tasks) default_args = ' "owner": "fluid",\n "retry_delay": timedelta(minutes=5),\n' return default_args, f" tasks = {{}}\n{nodes}\n{_edges(dag.tasks)}" @@ -1579,7 +1689,8 @@ def render_scheduled_dag( doc, description = _dag_doc(product_id, dag) default_args, body = _dag_tasks(dag) uses_assets = bool(dag.schedule.datasets) or any(t.outlets for t in dag.tasks) - script = "".join(f" {lit(line)},\n" for line in BASH_SCRIPT_LINES) + lines = KEEPALIVE_SCRIPT_LINES if dag.keepalive else BASH_SCRIPT_LINES + script = "".join(f" {lit(line)},\n" for line in lines) return ( f"{doc}\n" "\n" @@ -1697,6 +1808,10 @@ def render_fluid_apply_dags( "render_fluid_apply_dags", "render_scheduled_dag", "rendered_dag_of", + "KEEPALIVE_APPLY_ARGS", + "KEEPALIVE_SCHEDULE", + "KEEPALIVE_SCRIPT_LINES", + "KEEPALIVE_TASK_TIMEOUT_S", "runs_only_outside_schedulers", "schedule_scope_for", "scheduled_builds", diff --git a/tests/schedulers/fluid_apply_fixtures.py b/tests/schedulers/fluid_apply_fixtures.py index 7868caec..61e9537a 100644 --- a/tests/schedulers/fluid_apply_fixtures.py +++ b/tests/schedulers/fluid_apply_fixtures.py @@ -261,8 +261,23 @@ def rules_and_any() -> Dict[str, Any]: return _with_builds(contract, contract["builds"][0], staging, mart, notify, report) -#: Contracts that use ``builds[].dependencies`` and dataset triggers. +def kept_alive_stream() -> Dict[str, Any]: + """The Spark contract whose stream declares execution.retries: a keepalive DAG beside the + daily batch, on a cadence the build's Airflow DAG config sets.""" + contract = copy.deepcopy(SPARK_CONTRACT) + contract["builds"][0]["execution"] = { + "retries": {"maxAttempts": 5}, + "orchestration": { + "engine": "airflow", + "airflow": {"dagId": "events_keepalive", "dagConfig": {"schedule": "*/10 * * * *"}}, + }, + } + return contract + + +#: Contracts that use ``builds[].dependencies``, dataset triggers or a kept-alive stream. GRAPH_CASES: Dict[str, Tuple[Dict[str, Any], Optional[str], str]] = { "events_pipeline": (events_pipeline(), None, "contracts/events/contract.fluid.yaml"), "rules_and_any": (rules_and_any(), "aws", "contract.fluid.yaml"), + "kept_alive_stream": (kept_alive_stream(), "aws", "native.fluid.yaml"), } diff --git a/tests/schedulers/golden/fluid_apply/kept_alive_stream/daily_curated_dag.py.txt b/tests/schedulers/golden/fluid_apply/kept_alive_stream/daily_curated_dag.py.txt new file mode 100644 index 00000000..cbdee237 --- /dev/null +++ b/tests/schedulers/golden/fluid_apply/kept_alive_stream/daily_curated_dag.py.txt @@ -0,0 +1,123 @@ +"""Scheduled fluid build, generated by fluid. Edit the contract, not this file. + +Every run executes this on the Airflow worker, with the constants below +filled in (`--env` is left out when FLUID_ENV_NAME is empty): + + cd "$FLUID_PROJECT_DIR" + fluid apply "$FLUID_PROJECT_DIR/$CONTRACT_PATH" --env "$FLUID_ENV_NAME" \\ + --mode amend-and-build --build-id "$BUILD_ID" --yes + +Worker requirements: + +- FLUID_PROJECT_DIR names the directory that holds the product checkout, + the directory CI ran the pipeline from. +- `fluid` is on PATH, or FLUID_BIN names the executable. +- The credentials the apply needs are in the worker environment. Only + these reach the fluid process: + + PATH HOME USER LOGNAME LANG LANGUAGE LC_ALL LC_CTYPE TZ TMPDIR + SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE HTTP_PROXY + HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy GLUE_ROLE_ARN + S3_STAGING_DIR S3_DATA_DIR GOOG_SERVICE_ACCOUNT_NAME + TESTCONTAINERS_HOST_OVERRIDE PYTHON_DOTENV_DISABLED + + every name matching + + FLUID_* AWS_* GOOGLE_* GCP_* GCLOUD_* CLOUDSDK_* AZURE_* ARM_* + SNOWFLAKE_* DATABRICKS_* PG* POSTGRES_* REDSHIFT_* ATHENA_* VAULT_* + BAO_* DBT_* DLT_* DATAHUB_* DMM_* ODCS_* ODPS_* TF_* OPENLINEAGE_* + OTEL_* + + the variables the contract reads (CONTRACT_ENV_NAMES), and any names + listed, space separated, in FLUID_DAG_ENV_PASSTHROUGH. fluid starts with + nothing else, and never with a variable whose name starts AIRFLOW. +""" + +from datetime import timedelta + +import pendulum + +try: # Airflow 3 + from airflow.sdk import DAG +except ImportError: # Airflow 2.x + from airflow import DAG + +try: # Airflow 3, or Airflow 2 with apache-airflow-providers-standard + from airflow.providers.standard.operators.bash import BashOperator +except ImportError: # Airflow 2.x core operator + from airflow.operators.bash import BashOperator + +try: # Airflow 3 + from airflow.sdk import Asset +except ImportError: # Airflow 2.x + from airflow.datasets import Dataset as Asset + +PRODUCT_ID = 'platform.events' +BUILD_ID = 'daily-curated' +CONTRACT_PATH = 'native.fluid.yaml' +FLUID_ENV_NAME = 'aws' +CONTRACT_ENV_NAMES = '' +SCHEDULE = '0 2 * * *' +TIMEZONE = 'UTC' +RETRIES = 3 + +BASH_COMMAND = "\n".join( + [ + 'set -euo pipefail', + ': "${FLUID_PROJECT_DIR:?set FLUID_PROJECT_DIR on the Airflow worker to the directory that holds the product checkout}"', + 'keep=" PATH HOME USER LOGNAME LANG LANGUAGE LC_ALL LC_CTYPE TZ TMPDIR SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE HTTP_PROXY HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy GLUE_ROLE_ARN S3_STAGING_DIR S3_DATA_DIR GOOG_SERVICE_ACCOUNT_NAME TESTCONTAINERS_HOST_OVERRIDE PYTHON_DOTENV_DISABLED ${FLUID_DAG_CONTRACT_ENV:-} ${FLUID_DAG_ENV_PASSTHROUGH:-} "', + 'pass=()', + 'for name in $(compgen -e); do', + ' case "$name" in', + ' AIRFLOW*) continue ;;', + ' FLUID_*|AWS_*|GOOGLE_*|GCP_*|GCLOUD_*|CLOUDSDK_*|AZURE_*|ARM_*|SNOWFLAKE_*|DATABRICKS_*|PG*|POSTGRES_*|REDSHIFT_*|ATHENA_*|VAULT_*|BAO_*|DBT_*|DLT_*|DATAHUB_*|DMM_*|ODCS_*|ODPS_*|TF_*|OPENLINEAGE_*|OTEL_*) ;;', + ' *) case "$keep" in *" $name "*) ;; *) continue ;; esac ;;', + ' esac', + ' pass+=("$name=${!name}")', + 'done', + 'cd -- "$FLUID_PROJECT_DIR"', + 'contract="$PWD/$FLUID_DAG_CONTRACT"', + 'if [ ! -f "$contract" ]; then', + ' echo "fluid: no contract at $contract (FLUID_PROJECT_DIR + $FLUID_DAG_CONTRACT)" >&2', + ' exit 2', + 'fi', + 'set -- apply "$contract"', + 'if [ -n "$FLUID_DAG_ENV" ]; then set -- "$@" --env "$FLUID_DAG_ENV"; fi', + 'set -- "$@" --mode amend-and-build --build-id "$FLUID_DAG_BUILD_ID" --yes', + 'bin="${FLUID_BIN:-fluid}"', + 'case "$bin" in *=*) echo "fluid: FLUID_BIN may not contain =" >&2; exit 2 ;; esac', + 'exec env -i ${pass[@]+"${pass[@]}"} "$bin" "$@"', + ] +) + +with DAG( + dag_id='platform.events__aws__daily-curated', + description='fluid apply platform.events --build-id daily-curated', + schedule=SCHEDULE, + start_date=pendulum.datetime(2026, 1, 1, tz=TIMEZONE), + catchup=False, + max_active_runs=1, + default_args={ + "owner": "fluid", + "retries": RETRIES, + "retry_delay": timedelta(minutes=5), + "execution_timeout": timedelta(seconds=32400), + }, + tags=["fluid", PRODUCT_ID[:100]], + doc_md=__doc__, +) as dag: + BashOperator( + task_id="fluid_apply", + bash_command=BASH_COMMAND, + env={ + "FLUID_DAG_CONTRACT": CONTRACT_PATH, + "FLUID_DAG_ENV": FLUID_ENV_NAME, + "FLUID_DAG_BUILD_ID": BUILD_ID, + "FLUID_DAG_CONTRACT_ENV": CONTRACT_ENV_NAMES, + }, + append_env=True, + skip_on_exit_code=None, + outlets=[ + Asset('s3://lake/iceberg/curated/events_daily'), + ], + ) diff --git a/tests/schedulers/golden/fluid_apply/kept_alive_stream/stream_raw_dag.py.txt b/tests/schedulers/golden/fluid_apply/kept_alive_stream/stream_raw_dag.py.txt new file mode 100644 index 00000000..9b10a9b8 --- /dev/null +++ b/tests/schedulers/golden/fluid_apply/kept_alive_stream/stream_raw_dag.py.txt @@ -0,0 +1,121 @@ +"""Scheduled fluid build, generated by fluid. Edit the contract, not this file. + +Every run reconciles one always-on stream on the Airflow worker, with the +constants below filled in (`--env` is left out when FLUID_ENV_NAME is empty): + + cd "$FLUID_PROJECT_DIR" + fluid apply "$FLUID_PROJECT_DIR/$CONTRACT_PATH" --env "$FLUID_ENV_NAME" \\ + --mode reconcile --build-id "$BUILD_ID" --yes + +It starts the stream again when it stopped (a Spot interruption, a failed +driver) and changes nothing else; the build's execution.retries bound the +restarts, so the task itself never retries. It runs on SCHEDULE, one run +at a time, with no catch-up. The build's +execution.orchestration.airflow.dagConfig.schedule sets SCHEDULE. + +Worker requirements: + +- FLUID_PROJECT_DIR names the directory that holds the product checkout, + the directory CI ran the pipeline from. +- `fluid` is on PATH, or FLUID_BIN names the executable. +- The credentials the apply needs are in the worker environment. Only + these reach the fluid process: + + PATH HOME USER LOGNAME LANG LANGUAGE LC_ALL LC_CTYPE TZ TMPDIR + SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE HTTP_PROXY + HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy GLUE_ROLE_ARN + S3_STAGING_DIR S3_DATA_DIR GOOG_SERVICE_ACCOUNT_NAME + TESTCONTAINERS_HOST_OVERRIDE PYTHON_DOTENV_DISABLED + + every name matching + + FLUID_* AWS_* GOOGLE_* GCP_* GCLOUD_* CLOUDSDK_* AZURE_* ARM_* + SNOWFLAKE_* DATABRICKS_* PG* POSTGRES_* REDSHIFT_* ATHENA_* VAULT_* + BAO_* DBT_* DLT_* DATAHUB_* DMM_* ODCS_* ODPS_* TF_* OPENLINEAGE_* + OTEL_* + + the variables the contract reads (CONTRACT_ENV_NAMES), and any names + listed, space separated, in FLUID_DAG_ENV_PASSTHROUGH. fluid starts with + nothing else, and never with a variable whose name starts AIRFLOW. +""" + +from datetime import timedelta + +import pendulum + +try: # Airflow 3 + from airflow.sdk import DAG +except ImportError: # Airflow 2.x + from airflow import DAG + +try: # Airflow 3, or Airflow 2 with apache-airflow-providers-standard + from airflow.providers.standard.operators.bash import BashOperator +except ImportError: # Airflow 2.x core operator + from airflow.operators.bash import BashOperator + +PRODUCT_ID = 'platform.events' +BUILD_ID = 'stream-raw' +CONTRACT_PATH = 'native.fluid.yaml' +FLUID_ENV_NAME = 'aws' +CONTRACT_ENV_NAMES = '' +SCHEDULE = '*/10 * * * *' +TIMEZONE = 'UTC' +RETRIES = 0 + +BASH_COMMAND = "\n".join( + [ + 'set -euo pipefail', + ': "${FLUID_PROJECT_DIR:?set FLUID_PROJECT_DIR on the Airflow worker to the directory that holds the product checkout}"', + 'keep=" PATH HOME USER LOGNAME LANG LANGUAGE LC_ALL LC_CTYPE TZ TMPDIR SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE HTTP_PROXY HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy GLUE_ROLE_ARN S3_STAGING_DIR S3_DATA_DIR GOOG_SERVICE_ACCOUNT_NAME TESTCONTAINERS_HOST_OVERRIDE PYTHON_DOTENV_DISABLED ${FLUID_DAG_CONTRACT_ENV:-} ${FLUID_DAG_ENV_PASSTHROUGH:-} "', + 'pass=()', + 'for name in $(compgen -e); do', + ' case "$name" in', + ' AIRFLOW*) continue ;;', + ' FLUID_*|AWS_*|GOOGLE_*|GCP_*|GCLOUD_*|CLOUDSDK_*|AZURE_*|ARM_*|SNOWFLAKE_*|DATABRICKS_*|PG*|POSTGRES_*|REDSHIFT_*|ATHENA_*|VAULT_*|BAO_*|DBT_*|DLT_*|DATAHUB_*|DMM_*|ODCS_*|ODPS_*|TF_*|OPENLINEAGE_*|OTEL_*) ;;', + ' *) case "$keep" in *" $name "*) ;; *) continue ;; esac ;;', + ' esac', + ' pass+=("$name=${!name}")', + 'done', + 'cd -- "$FLUID_PROJECT_DIR"', + 'contract="$PWD/$FLUID_DAG_CONTRACT"', + 'if [ ! -f "$contract" ]; then', + ' echo "fluid: no contract at $contract (FLUID_PROJECT_DIR + $FLUID_DAG_CONTRACT)" >&2', + ' exit 2', + 'fi', + 'set -- apply "$contract"', + 'if [ -n "$FLUID_DAG_ENV" ]; then set -- "$@" --env "$FLUID_DAG_ENV"; fi', + 'set -- "$@" --mode reconcile --build-id "$FLUID_DAG_BUILD_ID" --yes', + 'bin="${FLUID_BIN:-fluid}"', + 'case "$bin" in *=*) echo "fluid: FLUID_BIN may not contain =" >&2; exit 2 ;; esac', + 'exec env -i ${pass[@]+"${pass[@]}"} "$bin" "$@"', + ] +) + +with DAG( + dag_id='platform.events__aws__stream-raw', + description='keep fluid stream platform.events stream-raw running', + schedule=SCHEDULE, + start_date=pendulum.datetime(2026, 1, 1, tz=TIMEZONE), + catchup=False, + max_active_runs=1, + default_args={ + "owner": "fluid", + "retries": RETRIES, + "retry_delay": timedelta(minutes=5), + "execution_timeout": timedelta(seconds=1800), + }, + tags=["fluid", PRODUCT_ID[:100]], + doc_md=__doc__, +) as dag: + BashOperator( + task_id="fluid_keepalive", + bash_command=BASH_COMMAND, + env={ + "FLUID_DAG_CONTRACT": CONTRACT_PATH, + "FLUID_DAG_ENV": FLUID_ENV_NAME, + "FLUID_DAG_BUILD_ID": BUILD_ID, + "FLUID_DAG_CONTRACT_ENV": CONTRACT_ENV_NAMES, + }, + append_env=True, + skip_on_exit_code=None, + ) diff --git a/tests/schedulers/test_fluid_apply_airflow_live.py b/tests/schedulers/test_fluid_apply_airflow_live.py index d7493c9d..703a467d 100644 --- a/tests/schedulers/test_fluid_apply_airflow_live.py +++ b/tests/schedulers/test_fluid_apply_airflow_live.py @@ -48,6 +48,7 @@ "edges": sorted([t.task_id, d] for t in dag.tasks for d in t.downstream_task_ids), "rules": {t.task_id: getattr(t.trigger_rule, "value", t.trigger_rule) for t in dag.tasks}, "outlets": {t.task_id: [o.uri for o in t.outlets] for t in dag.tasks}, + "retries": {t.task_id: t.retries for t in dag.tasks}, "timetable": type(dag.timetable).__name__, } print(json.dumps(out)) @@ -63,7 +64,8 @@ def _expected(dag: fluid_apply.ScheduledDag, major: str) -> Dict[str, Any]: only = len(dag.tasks) == 1 - task_id = {t.build_id: ("fluid_apply" if only else t.build_id) for t in dag.tasks} + single = "fluid_keepalive" if dag.keepalive else "fluid_apply" + task_id = {t.build_id: (single if only else t.build_id) for t in dag.tasks} if dag.schedule.datasets and dag.schedule.cron is not None: timetable = _OR_TIME[major] else: @@ -73,6 +75,7 @@ def _expected(dag: fluid_apply.ScheduledDag, major: str) -> Dict[str, Any]: "edges": sorted([task_id[u], task_id[t.build_id]] for t in dag.tasks for u in t.upstream), "rules": {task_id[t.build_id]: t.trigger_rule or "all_success" for t in dag.tasks}, "outlets": {task_id[t.build_id]: list(t.outlets) for t in dag.tasks}, + "retries": {task_id[t.build_id]: t.retries for t in dag.tasks}, "timetable": timetable, } diff --git a/tests/schedulers/test_fluid_apply_keepalive.py b/tests/schedulers/test_fluid_apply_keepalive.py new file mode 100644 index 00000000..6a21d874 --- /dev/null +++ b/tests/schedulers/test_fluid_apply_keepalive.py @@ -0,0 +1,150 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""An always-on Spark stream that declares ``execution.retries`` gets a keepalive DAG. + +Its one task runs the reconcile-only ``fluid apply`` (:data:`fluid_apply.KEEPALIVE_APPLY_ARGS`) +for that stream every few minutes, so a stream a Spot interruption stopped starts again. One +run at a time, no catch-up, no Airflow retries: the retry budget is the build's own. +""" + +from __future__ import annotations + +import copy +from datetime import timedelta +from pathlib import Path +from typing import Any, Dict + +import pytest + +from fluid_build.schedulers.airflow import fluid_apply +from tests.cli._schedule_dag_fixtures import load_dag, run_task +from tests.schedulers.fluid_apply_fixtures import SPARK_CONTRACT + + +def _contract(**execution: Any) -> Dict[str, Any]: + contract = copy.deepcopy(SPARK_CONTRACT) + contract["builds"][0]["execution"] = {"retries": {"maxAttempts": 5}, **execution} + return contract + + +def _files(contract: Dict[str, Any], env: Any = "aws") -> Dict[str, str]: + return fluid_apply.render_fluid_apply_dags(contract, env=env, contract_path="native.fluid.yaml") + + +def _keepalive_config(schedule: Any) -> Dict[str, Any]: + return { + "orchestration": { + "engine": "airflow", + "airflow": {"dagId": "events_keepalive", "dagConfig": {"schedule": schedule}}, + } + } + + +@pytest.mark.parametrize("airflow3", [True, False]) +def test_a_stream_with_retries_gets_a_keepalive_dag(monkeypatch, airflow3: bool): + files = _files(_contract()) + assert sorted(files) == ["daily_curated_dag.py", "stream_raw_dag.py"] + loaded = load_dag(files["stream_raw_dag.py"], monkeypatch, airflow3=airflow3) + assert loaded.dag["dag_id"] == "platform.events__aws__stream-raw" + assert loaded.dag["schedule"] == fluid_apply.KEEPALIVE_SCHEDULE + assert loaded.dag["max_active_runs"] == 1 and loaded.dag["catchup"] is False + assert loaded.dag["default_args"]["retries"] == 0 + assert loaded.dag["default_args"]["execution_timeout"] == timedelta( + seconds=fluid_apply.KEEPALIVE_TASK_TIMEOUT_S + ) + (task,) = loaded.tasks + assert task["task_id"] == "fluid_keepalive" + assert task["env"]["FLUID_DAG_BUILD_ID"] == "stream-raw" + assert "outlets" not in task + assert task["bash_command"] == "\n".join(fluid_apply.KEEPALIVE_SCRIPT_LINES) + + +def test_each_run_is_the_reconcile_only_apply_of_that_stream(tmp_path: Path, monkeypatch): + project = tmp_path / "project" + project.mkdir() + (project / "native.fluid.yaml").write_text("id: x\n", encoding="utf-8") + (task,) = load_dag(_files(_contract())["stream_raw_dag.py"], monkeypatch).tasks + run = run_task(task, tmp_path, {"FLUID_PROJECT_DIR": str(project)}) + assert run.returncode == 0, run.stderr + assert run.argv is not None + assert run.argv[2:] == [ + "--env", + "aws", + *fluid_apply.KEEPALIVE_APPLY_ARGS, + "--build-id", + "stream-raw", + "--yes", + ] + + +def test_the_reconcile_arguments_live_in_one_constant(): + tail = [line for line in fluid_apply.KEEPALIVE_SCRIPT_LINES if "--build-id" in line] + assert tail == [ + 'set -- "$@" ' + + " ".join(fluid_apply.KEEPALIVE_APPLY_ARGS) + + ' --build-id "$FLUID_DAG_BUILD_ID" --yes' + ] + # The keepalive script is the apply script but for that line, and holds no contract text. + differ = set(fluid_apply.KEEPALIVE_SCRIPT_LINES) ^ set(fluid_apply.BASH_SCRIPT_LINES) + assert len(differ) == 2 + for word in fluid_apply.KEEPALIVE_APPLY_ARGS: + assert word.replace("-", "").isalnum(), word + + +def test_without_retries_a_stream_still_gets_no_dag(): + assert sorted(_files(copy.deepcopy(SPARK_CONTRACT))) == ["daily_curated_dag.py"] + + +def test_dag_config_schedule_sets_the_cadence(monkeypatch): + source = _files(_contract(**_keepalive_config("*/2 * * * *")))["stream_raw_dag.py"] + assert load_dag(source, monkeypatch).dag["schedule"] == "*/2 * * * *" + + +@pytest.mark.parametrize( + "schedule, message", + [ + ("@once", "a keepalive DAG reconciles the stream on a recurring cron"), + ("*/90 * * *", "is not a five-field cron expression"), + ], +) +def test_a_cadence_airflow_cannot_run_is_refused(schedule: str, message: str): + with pytest.raises(fluid_apply.ScheduleRenderError) as exc: + _files(_contract(**_keepalive_config(schedule))) + assert "execution.orchestration.airflow.dagConfig.schedule" in str(exc.value) + assert message in str(exc.value) + + +@pytest.mark.parametrize("engine", ["none", "dagster"]) +def test_a_stream_on_another_engine_gets_no_keepalive(engine: str): + contract = _contract(orchestration={"engine": engine}) + assert sorted(_files(contract)) == ["daily_curated_dag.py"] + + +def test_a_stream_that_runs_to_completion_is_a_batch_not_a_keepalive(): + contract = _contract() + contract["builds"][0]["properties"]["stream"]["trigger"] = {"availableNow": True} + contract["builds"][0]["execution"]["trigger"] = {"type": "schedule", "schedule": "@hourly"} + source = _files(contract)["stream_raw_dag.py"] + assert 'task_id="fluid_apply"' in source and "fluid_keepalive" not in source + + +def test_a_streams_only_contract_with_retries_needs_a_dag(): + contract = _contract() + contract["builds"] = contract["builds"][:1] + assert fluid_apply.has_scheduled_builds(contract) + assert not fluid_apply.runs_only_outside_schedulers(contract) + assert sorted(_files(contract)) == ["stream_raw_dag.py"] + + +def test_a_batch_still_does_not_wait_for_the_stream_it_depends_on(): + contract = _contract() + contract["builds"][1]["dependencies"] = [{"buildId": "stream-raw"}] + (daily,) = [d for d in fluid_apply.scheduled_dags(contract) if not d.keepalive] + assert [t.build_id for t in daily.tasks] == ["daily-curated"] + assert daily.unwaited[0][0] == "stream-raw" + + +def test_the_keepalive_reads_back_as_the_products_dag(tmp_path: Path): + (tmp_path / "stream_raw_dag.py").write_text(_files(_contract())["stream_raw_dag.py"]) + assert fluid_apply.rendered_dag_of(tmp_path / "stream_raw_dag.py") == ("platform.events", "aws") From e28f5eb367938d351bf718bf3bff0808123cfd68 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:52:57 +0100 Subject: [PATCH 02/10] feat(schedule): a keepalive runs fluid apply --mode reconcile The keepalive DAG's task runs fluid apply [--env] --mode reconcile --build-id --yes. That runs no tofu apply: it starts the stream again on its checkpoint within the restart budget the build's execution.retries sets, and fails the run when the stream drifted from the contract or the budget is spent. The arguments stay in KEEPALIVE_APPLY_ARGS, joined one space apart whatever they hold, and a test parses them with fluid's own parser and its --build-id check, so the DAG cannot run a mode or flag that fluid apply refuses. --- AGENTS.md | 2 +- CHANGELOG.md | 14 ++++--- fluid_build/schedulers/airflow/fluid_apply.py | 26 ++++++++----- .../kept_alive_stream/stream_raw_dag.py.txt | 9 +++-- .../schedulers/test_fluid_apply_keepalive.py | 39 ++++++++++++++----- 5 files changed, 60 insertions(+), 30 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 74fba4c2..25841076 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -154,7 +154,7 @@ forge-cli/ | Provider plugins (planning) | `fluid_build/providers/{local,gcp,aws,snowflake,odps,odcs}/` — `plan()` emits abstract actions | | **IaC emitter (apply)** | `fluid_build/iac/` — modular OpenTofu plugin registry. `cli/_apply_opentofu_engine.py` compiles the contract to `main.tf.json` and shells `tofu` for cloud providers. `cli/generate_iac.py` is `fluid generate iac` for review-only emit. One plugin per cloud at `iac/providers/{aws,gcp,snowflake}.py` (dbt-adapter pattern). | | Path-B schedulers | `fluid_build/providers/aws/plan/schedule.py` (EventBridge/MWAA/Lambda/Step-Functions via stage 6 `SchedulePlanner`) | -| Airflow DAGs | `fluid_build/schedulers/airflow/` is the one renderer. `render_dag()` (`__init__.py`, operators in `_operators.py`) renders a contract's `orchestration` section for gcp/aws/snowflake; `fluid_apply.py` renders each build as one `fluid apply --mode amend-and-build --build-id` task, a forge Spark build included (its build phase starts the run and waits; a stream that is not `availableNow` gets no DAG). Builds that `builds[].dependencies` link share one DAG (`scheduled_dags()`: the graph and the schedule; `render_build_task()`: the one function that writes a build's node), on the one trigger they declare: a cron, `trigger.datasets` (`dataset`), or both (`schedule_and_dataset`). Every task declares its build's exposes as Airflow asset `outlets`, named as Airflow's providers name the storage (`_assets.py`). A Spark stream that declares `execution.retries` gets a keepalive DAG whose task runs the reconcile-only apply (`KEEPALIVE_APPLY_ARGS`). `fluid generate schedule`, `fluid export --engine airflow` and `fluid generate-airflow` (a deprecated alias of the first) all come through it. Task variables go through `codegen_utils.unique_task_identifiers`. | +| Airflow DAGs | `fluid_build/schedulers/airflow/` is the one renderer. `render_dag()` (`__init__.py`, operators in `_operators.py`) renders a contract's `orchestration` section for gcp/aws/snowflake; `fluid_apply.py` renders each build as one `fluid apply --mode amend-and-build --build-id` task, a forge Spark build included (its build phase starts the run and waits; a stream that is not `availableNow` gets no DAG). Builds that `builds[].dependencies` link share one DAG (`scheduled_dags()`: the graph and the schedule; `render_build_task()`: the one function that writes a build's node), on the one trigger they declare: a cron, `trigger.datasets` (`dataset`), or both (`schedule_and_dataset`). Every task declares its build's exposes as Airflow asset `outlets`, named as Airflow's providers name the storage (`_assets.py`). A Spark stream that declares `execution.retries` gets a keepalive DAG whose task runs `fluid apply --mode reconcile --build-id ` every 5 minutes (`KEEPALIVE_APPLY_ARGS`). `fluid generate schedule`, `fluid export --engine airflow` and `fluid generate-airflow` (a deprecated alias of the first) all come through it. Task variables go through `codegen_utils.unique_task_identifiers`. | | Contract schemas | `fluid_build/schemas/*.json` (0.7.2 is latest; `SchemaManager.latest_bundled_version()` is the dynamic lookup) | | Policy engine | `fluid_build/policy/` (compiler, agent_policy, sovereignty, guardrails) | | Plan-binding helpers | `fluid_build/forge/core/plan_digest.py` — `inject_digests`, `verify_plan_binding`, `PlanBindingError` | diff --git a/CHANGELOG.md b/CHANGELOG.md index 08f74e19..f6639877 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -118,12 +118,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **A keepalive DAG for an always-on stream that declares `execution.retries`.** `fluid generate schedule` gives each forge Spark stream that is not `trigger.availableNow` and - declares `execution.retries` a small DAG whose one task runs the reconcile-only `fluid apply` - for that stream, so a stream a Spot interruption or a failed driver stopped starts again. It - runs every 5 minutes, or on the build's `execution.orchestration.airflow.dagConfig.schedule`, - one run at a time, with no catch-up and no Airflow retries: the build's own retries bound the - restarts. The task declares no outlets. A contract whose only builds are such streams now gets - this DAG instead of "No build needs a DAG". + declares `execution.retries` a small DAG whose one task runs + `fluid apply --mode reconcile --build-id --yes`, so a stream a Spot + interruption or a failed driver stopped starts again on its checkpoint. It runs every 5 + minutes, or on the build's `execution.orchestration.airflow.dagConfig.schedule`, one run at a + time, with no catch-up and no Airflow retries: the build's `execution.retries` is the restart + budget, and a run fails when the stream drifted from the contract or the budget is spent. The + task declares no outlets. A contract whose only builds are such streams now gets this DAG + instead of "No build needs a DAG". - **`fluid generate schedule` renders the build graph: `builds[].dependencies` and dataset triggers.** Builds that `builds[].dependencies` link now run in one Airflow DAG, one diff --git a/fluid_build/schedulers/airflow/fluid_apply.py b/fluid_build/schedulers/airflow/fluid_apply.py index c2721735..ecef183c 100644 --- a/fluid_build/schedulers/airflow/fluid_apply.py +++ b/fluid_build/schedulers/airflow/fluid_apply.py @@ -125,9 +125,10 @@ APPLY_MODE = "amend-and-build" #: What a keepalive DAG passes to ``fluid apply`` after the contract and ``--env``, before -#: ``--build-id --yes``: reconcile that one stream, restarting it when it stopped, and -#: change nothing else. The one place the reconcile-only apply is named. PLACEHOLDER: the -#: reconcile-only apply path does not exist yet; set the real arguments here once it does. +#: ``--build-id --yes``. ``--mode reconcile`` runs no ``tofu apply``: it starts that one +#: stream again on its checkpoint when it stopped, within the restart budget the build's +#: ``execution.retries`` sets, and exits non-zero when the stream drifted from the contract or +#: the budget is spent. The one place a keepalive's apply arguments are named. KEEPALIVE_APPLY_ARGS: Tuple[str, ...] = ("--mode", "reconcile") #: How often a keepalive DAG reconciles its stream, unless the build's @@ -1358,10 +1359,16 @@ def rendered_dag_of(path: Path) -> Optional[Tuple[str, str]]: BASH_SCRIPT_LINES = _SCRIPT_HEAD + _APPLY_TAIL + _SCRIPT_END + +def _apply_args(args: Tuple[str, ...], build_ref: str) -> str: + """``args``, then ``--build-id --yes``: one space apart, however many ``args``.""" + return " ".join((*args, "--build-id", build_ref, "--yes")) + + _KEEPALIVE_TAIL = ( 'set -- apply "$contract"', 'if [ -n "$FLUID_DAG_ENV" ]; then set -- "$@" --env "$FLUID_DAG_ENV"; fi', - 'set -- "$@" ' + " ".join(KEEPALIVE_APPLY_ARGS) + ' --build-id "$FLUID_DAG_BUILD_ID" --yes', + 'set -- "$@" ' + _apply_args(KEEPALIVE_APPLY_ARGS, '"$FLUID_DAG_BUILD_ID"'), ) #: The script of a keepalive DAG's task: :data:`BASH_SCRIPT_LINES` but for the apply arguments. @@ -1412,12 +1419,13 @@ def _wrap(words: List[str], indent: str, width: int = 76) -> str: "\n" ' cd "$FLUID_PROJECT_DIR"\n' ' fluid apply "$FLUID_PROJECT_DIR/$CONTRACT_PATH" --env "$FLUID_ENV_NAME" \\\\\n' - f' {" ".join(KEEPALIVE_APPLY_ARGS)} --build-id "$BUILD_ID" --yes\n' + " " + _apply_args(KEEPALIVE_APPLY_ARGS, '"$BUILD_ID"') + "\n" "\n" - "It starts the stream again when it stopped (a Spot interruption, a failed\n" - "driver) and changes nothing else; the build's execution.retries bound the\n" - "restarts, so the task itself never retries. It runs on SCHEDULE, one run\n" - "at a time, with no catch-up. The build's\n" + "It runs no tofu apply. It starts the stream again on its checkpoint when it\n" + "stopped (a Spot interruption, a failed driver), within the restart budget the\n" + "build's execution.retries sets, and fails the run when the stream drifted\n" + "from the contract or the budget is spent, so the task itself never retries.\n" + "It runs on SCHEDULE, one run at a time, with no catch-up. The build's\n" "execution.orchestration.airflow.dagConfig.schedule sets SCHEDULE.\n" ) diff --git a/tests/schedulers/golden/fluid_apply/kept_alive_stream/stream_raw_dag.py.txt b/tests/schedulers/golden/fluid_apply/kept_alive_stream/stream_raw_dag.py.txt index 9b10a9b8..f225c5fe 100644 --- a/tests/schedulers/golden/fluid_apply/kept_alive_stream/stream_raw_dag.py.txt +++ b/tests/schedulers/golden/fluid_apply/kept_alive_stream/stream_raw_dag.py.txt @@ -7,10 +7,11 @@ constants below filled in (`--env` is left out when FLUID_ENV_NAME is empty): fluid apply "$FLUID_PROJECT_DIR/$CONTRACT_PATH" --env "$FLUID_ENV_NAME" \\ --mode reconcile --build-id "$BUILD_ID" --yes -It starts the stream again when it stopped (a Spot interruption, a failed -driver) and changes nothing else; the build's execution.retries bound the -restarts, so the task itself never retries. It runs on SCHEDULE, one run -at a time, with no catch-up. The build's +It runs no tofu apply. It starts the stream again on its checkpoint when it +stopped (a Spot interruption, a failed driver), within the restart budget the +build's execution.retries sets, and fails the run when the stream drifted +from the contract or the budget is spent, so the task itself never retries. +It runs on SCHEDULE, one run at a time, with no catch-up. The build's execution.orchestration.airflow.dagConfig.schedule sets SCHEDULE. Worker requirements: diff --git a/tests/schedulers/test_fluid_apply_keepalive.py b/tests/schedulers/test_fluid_apply_keepalive.py index 6a21d874..0091a2c0 100644 --- a/tests/schedulers/test_fluid_apply_keepalive.py +++ b/tests/schedulers/test_fluid_apply_keepalive.py @@ -3,9 +3,10 @@ """An always-on Spark stream that declares ``execution.retries`` gets a keepalive DAG. -Its one task runs the reconcile-only ``fluid apply`` (:data:`fluid_apply.KEEPALIVE_APPLY_ARGS`) -for that stream every few minutes, so a stream a Spot interruption stopped starts again. One -run at a time, no catch-up, no Airflow retries: the retry budget is the build's own. +Its one task runs ``fluid apply --mode reconcile --build-id `` +(:data:`fluid_apply.KEEPALIVE_APPLY_ARGS`) every few minutes, so a stream a Spot interruption +stopped starts again. One run at a time, no catch-up, no Airflow retries: the restart budget is +the build's own ``execution.retries``. """ from __future__ import annotations @@ -60,7 +61,7 @@ def test_a_stream_with_retries_gets_a_keepalive_dag(monkeypatch, airflow3: bool) assert task["bash_command"] == "\n".join(fluid_apply.KEEPALIVE_SCRIPT_LINES) -def test_each_run_is_the_reconcile_only_apply_of_that_stream(tmp_path: Path, monkeypatch): +def test_each_run_reconciles_that_one_stream(tmp_path: Path, monkeypatch): project = tmp_path / "project" project.mkdir() (project / "native.fluid.yaml").write_text("id: x\n", encoding="utf-8") @@ -78,13 +79,10 @@ def test_each_run_is_the_reconcile_only_apply_of_that_stream(tmp_path: Path, mon ] -def test_the_reconcile_arguments_live_in_one_constant(): +def test_the_keepalive_runs_apply_in_reconcile_mode(): + assert fluid_apply.KEEPALIVE_APPLY_ARGS == ("--mode", "reconcile") tail = [line for line in fluid_apply.KEEPALIVE_SCRIPT_LINES if "--build-id" in line] - assert tail == [ - 'set -- "$@" ' - + " ".join(fluid_apply.KEEPALIVE_APPLY_ARGS) - + ' --build-id "$FLUID_DAG_BUILD_ID" --yes' - ] + assert tail == ['set -- "$@" --mode reconcile --build-id "$FLUID_DAG_BUILD_ID" --yes'] # The keepalive script is the apply script but for that line, and holds no contract text. differ = set(fluid_apply.KEEPALIVE_SCRIPT_LINES) ^ set(fluid_apply.BASH_SCRIPT_LINES) assert len(differ) == 2 @@ -92,6 +90,27 @@ def test_the_reconcile_arguments_live_in_one_constant(): assert word.replace("-", "").isalnum(), word +def test_apply_arguments_render_one_space_apart_even_when_there_are_none(): + assert fluid_apply._apply_args((), '"$B"') == '--build-id "$B" --yes' + assert fluid_apply._apply_args(("--mode", "x"), '"$B"') == '--mode x --build-id "$B" --yes' + source = _files(_contract())["stream_raw_dag.py"] + assert " --build-id" not in source and "reconcile " not in source + + +def test_fluid_apply_accepts_the_keepalive_argv(): + """The DAG never runs an apply the CLI would refuse: the mode exists, and it takes --build-id.""" + from fluid_build.cli import build_parser + from fluid_build.cli.apply import _check_build_id_mode + from fluid_build.forge.core.apply_modes import parse_mode + + argv = ["apply", "native.fluid.yaml", "--env", "aws", *fluid_apply.KEEPALIVE_APPLY_ARGS] + args = build_parser().parse_args([*argv, "--build-id", "stream-raw", "--yes"]) + assert (args.build_id, args.yes) == ("stream-raw", True) + mode = parse_mode(args.mode) + assert mode.value == "reconcile" + _check_build_id_mode(args.build_id, mode) + + def test_without_retries_a_stream_still_gets_no_dag(): assert sorted(_files(copy.deepcopy(SPARK_CONTRACT))) == ["daily_curated_dag.py"] From 9333ee8b617d2e38258bebcaa4085f2d241d8690 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 19 Nov 2026 00:06:30 +0100 Subject: [PATCH 03/10] refactor(schedule): build keepalive DAGs beside their helpers A keepalive DAG reads only the contract's builds: a stream is in no build graph, since nothing waits for it. So _keepalive_dags() sits with _keeps_alive() and _keepalive_schedule() rather than in _BuildGraph, which keeps to the dependencies between the builds a DAG runs. --- fluid_build/schedulers/airflow/fluid_apply.py | 54 ++++++++++--------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/fluid_build/schedulers/airflow/fluid_apply.py b/fluid_build/schedulers/airflow/fluid_apply.py index ecef183c..b64630b4 100644 --- a/fluid_build/schedulers/airflow/fluid_apply.py +++ b/fluid_build/schedulers/airflow/fluid_apply.py @@ -490,6 +490,34 @@ def _trigger_schedule(build: Dict[str, Any], managed: bool = True) -> Optional[D return trigger if kind == _CRON else None +def _keepalive_dags( + contract: Dict[str, Any], builds: List[Dict[str, Any]], managed: bool +) -> List[ScheduledDag]: + """One keepalive DAG per always-on stream that declares ``execution.retries``. + + Its one task runs ``fluid apply --mode reconcile`` for the stream every few minutes, so a + stream a Spot interruption or a failed driver stopped starts again, within the restart + budget the build's ``execution.retries`` sets. A stream is in no build graph (nothing waits + for it), so this reads the builds alone. It has no outlets: a stream updates its tables all + the time, and an asset event on every run would start each DAG scheduled on them. + """ + out: List[ScheduledDag] = [] + for build in builds: + if not _keeps_alive(build, managed): + continue + build_id = _task_build_id(build) + _check_spark_ids(contract, build_id) + out.append( + ScheduledDag( + build_id=build_id, + schedule=DagSchedule(cron=_keepalive_schedule(build), timezone="UTC"), + tasks=(BuildTask(build_id, 0, KEEPALIVE_TASK_TIMEOUT_S),), + keepalive=True, + ) + ) + return out + + def has_scheduled_builds(contract: Dict[str, Any]) -> bool: """True when at least one build declares a trigger a DAG runs on (no validation): a cron, datasets, or both.""" @@ -1043,30 +1071,6 @@ def check_expose_refs(self, dags: List[ScheduledDag]) -> None: "schedule (trigger.type: schedule)" ) - def keepalives(self) -> List[ScheduledDag]: - """One keepalive DAG per always-on stream that declares ``execution.retries``. - - Its one task reconciles the stream every few minutes, so a stream a Spot interruption - or a failed driver stopped starts again; the build's own retries bound how often. It - has no outlets: a stream updates its tables all the time, and an asset event on every - run would start each DAG scheduled on them. - """ - out: List[ScheduledDag] = [] - for build in self.builds: - if not _keeps_alive(build, self.managed): - continue - build_id = _task_build_id(build) - _check_spark_ids(self.contract, build_id) - out.append( - ScheduledDag( - build_id=build_id, - schedule=DagSchedule(cron=_keepalive_schedule(build), timezone="UTC"), - tasks=(BuildTask(build_id, 0, KEEPALIVE_TASK_TIMEOUT_S),), - keepalive=True, - ) - ) - return out - def _why_no_task(self, build: Dict[str, Any]) -> str: kind, trigger = _classify(build, self.managed) if kind == _OUTSIDE: @@ -1135,7 +1139,7 @@ def scheduled_dags(contract: Dict[str, Any]) -> List[ScheduledDag]: return [] graph = _BuildGraph(contract, [b for b in raw_builds if isinstance(b, dict)]) dags = [dag for members in graph.groups() if (dag := graph.dag(members)) is not None] - dags += graph.keepalives() + dags += _keepalive_dags(contract, graph.builds, graph.managed) seen: Set[str] = set() for task in (t for dag in dags for t in dag.tasks): if task.build_id in seen: From dd775836be8af73c3a604871b31e64bb0f2099b3 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 19 Nov 2026 00:27:54 +0100 Subject: [PATCH 04/10] fix(schedule): count a keepalive only where it runs, and as updating nothing Four places counted a keepalive DAG as more than it is: - A dataset trigger on an expose that only a kept-alive stream writes was no longer refused, though a keepalive declares no outlets and the trigger would never fire. It is refused again, and the message says why. - fluid generate schedule --scheduler prefect or dagster wrote a placeholder flow for a contract whose only builds are kept-alive streams, and fluid scaffold-composer wrote a daily plan-and-apply DAG for one. Neither renders a keepalive. Both write nothing again, and name the Airflow keepalive DAG. - A stream that trigger.type: continuous or a Flink job leaves to another platform got a keepalive DAG. It no longer does, as _classify already has it. --- CHANGELOG.md | 6 ++- fluid_build/cli/generate_schedule.py | 12 ++++- fluid_build/cli/scaffold_composer.py | 20 ++++++-- fluid_build/schedulers/airflow/fluid_apply.py | 29 +++++++++--- .../cli/test_generate_schedule_build_graph.py | 47 +++++++++++++++++++ .../schedulers/test_fluid_apply_keepalive.py | 23 +++++++++ 6 files changed, 121 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6639877..da520624 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -124,8 +124,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 minutes, or on the build's `execution.orchestration.airflow.dagConfig.schedule`, one run at a time, with no catch-up and no Airflow retries: the build's `execution.retries` is the restart budget, and a run fails when the stream drifted from the contract or the budget is spent. The - task declares no outlets. A contract whose only builds are such streams now gets this DAG - instead of "No build needs a DAG". + task declares no outlets, so a dataset trigger on what only the stream writes is refused, as + before. A contract whose only builds are such streams now gets this DAG on Airflow instead of + "No build needs a DAG"; on Prefect or Dagster, and in `fluid scaffold-composer`, it still has + nothing to schedule, and the message names the Airflow keepalive. - **`fluid generate schedule` renders the build graph: `builds[].dependencies` and dataset triggers.** Builds that `builds[].dependencies` link now run in one Airflow DAG, one diff --git a/fluid_build/cli/generate_schedule.py b/fluid_build/cli/generate_schedule.py index 5a5df7e5..0e4d0424 100644 --- a/fluid_build/cli/generate_schedule.py +++ b/fluid_build/cli/generate_schedule.py @@ -170,8 +170,11 @@ def run(args: Any, logger: logging.Logger) -> int: # stream fluid apply keeps running, or a build whose trigger is # continuous) has nothing to schedule, which is not an error: unless it # hand-declares orchestration tasks, or names a scheduler that does not - # exist. - if fluid_apply.runs_only_outside_schedulers(contract) and not ( + # exist. Only the Airflow DAGs keep a stream running (its keepalive DAG), + # so on another scheduler a stream that declares execution.retries has + # nothing to schedule either. + keepalive = (scheduler_name or fluid_apply.DEFAULT_ENGINE) == fluid_apply.DEFAULT_ENGINE + if fluid_apply.runs_only_outside_schedulers(contract, keepalive=keepalive) and not ( fluid_apply.has_explicit_tasks(contract) ): from fluid_build.schedulers import has_scheduler @@ -183,6 +186,11 @@ def run(args: Any, logger: logging.Logger) -> int: "continuously (trigger.type: continuous), so there is nothing to " "schedule. No files written." ) + if not fluid_apply.runs_only_outside_schedulers(contract): + cprint( + "A stream that declares execution.retries gets a keepalive DAG on " + "Airflow: fluid generate schedule --scheduler airflow." + ) return 0 if not scheduler_name: diff --git a/fluid_build/cli/scaffold_composer.py b/fluid_build/cli/scaffold_composer.py index 432597cd..13a2a608 100644 --- a/fluid_build/cli/scaffold_composer.py +++ b/fluid_build/cli/scaffold_composer.py @@ -202,16 +202,23 @@ def _wrote_dag(dag_path: str, dag_id: str) -> bool: ) -def _report_nothing_to_schedule(dag_path: str, dag_id: str) -> None: +def _report_nothing_to_schedule(dag_path: str, dag_id: str, *, keepalive: bool = False) -> None: """Say why no DAG is written, as ``fluid generate schedule`` does, and name a DAG this command wrote for the product before: left in place, it keeps scheduling the product wherever it was deployed. A file of the same name - that it did not write is someone else's, and is not mentioned.""" + that it did not write is someone else's, and is not mentioned. With + ``keepalive``, a stream declares ``execution.retries``: say which command + writes the DAG that keeps it running.""" cprint( "Every build runs outside forge-cli (a Flink job, for example), as a stream that fluid " "apply keeps running, or continuously (trigger.type: continuous), so there is nothing " "to schedule. No DAG written." ) + if keepalive: + cprint( + "A stream that declares execution.retries gets a keepalive DAG from fluid generate " + "schedule, which restarts it when it stops." + ) if _wrote_dag(dag_path, dag_id): # Printed as written: soft_wrap leaves wrapping to the terminal, so the # path is never broken by an inserted newline and copies whole, and no @@ -241,15 +248,18 @@ def run(args, logger: logging.Logger) -> int: # apply keeps running, so a contract with no other build has nothing to # schedule and gets no DAG. The test is `fluid generate schedule`'s, # hand-declared orchestration tasks included, so the two commands agree - # on such a contract. + # on such a contract. A stream's keepalive DAG is generate schedule's + # alone: this command's validate, plan and apply DAG cannot keep a + # stream running, so here a stream with execution.retries is outside too. from fluid_build.schedulers.airflow.fluid_apply import ( has_explicit_tasks, runs_only_outside_schedulers, ) - if runs_only_outside_schedulers(c) and not has_explicit_tasks(c): + if runs_only_outside_schedulers(c, keepalive=False) and not has_explicit_tasks(c): dag_path = os.path.join(os.path.abspath(args.out_dir), f"{dag_id}.py") - _report_nothing_to_schedule(dag_path, dag_id) + kept_alive = not runs_only_outside_schedulers(c) + _report_nothing_to_schedule(dag_path, dag_id, keepalive=kept_alive) return 0 cron = _validate_cron(_extract_cron(c)) # SECURITY (shell + Python-literal injection): the contract path is diff --git a/fluid_build/schedulers/airflow/fluid_apply.py b/fluid_build/schedulers/airflow/fluid_apply.py index b64630b4..2336f744 100644 --- a/fluid_build/schedulers/airflow/fluid_apply.py +++ b/fluid_build/schedulers/airflow/fluid_apply.py @@ -452,8 +452,10 @@ def _outside_why(build: Dict[str, Any], managed: bool) -> str: def _keeps_alive(build: Dict[str, Any], managed: bool) -> bool: """Whether a keepalive DAG watches the build: a forge Spark stream (one that is not - ``trigger.availableNow``) that declares ``execution.retries``, on Airflow's engine.""" - if _spark_kind(build, managed) != "streaming": + ``trigger.availableNow``) that declares ``execution.retries``, on Airflow's engine. A build + another platform runs (:func:`is_unscheduled_build`: ``trigger.type: continuous``, a Flink + job) is that platform's to keep running, as :func:`_classify` has it.""" + if is_unscheduled_build(build) or _spark_kind(build, managed) != "streaming": return False execution = build.get("execution") if not isinstance(execution, dict) or not isinstance(execution.get("retries"), dict): @@ -532,15 +534,20 @@ def has_scheduled_builds(contract: Dict[str, Any]) -> bool: ) -def runs_only_outside_schedulers(contract: Dict[str, Any]) -> bool: +def runs_only_outside_schedulers(contract: Dict[str, Any], *, keepalive: bool = True) -> bool: """True when the contract has builds and no scheduler starts any of them: each is a stream - ``fluid apply`` keeps running, runs continuously, or runs on its own platform.""" + ``fluid apply`` keeps running, runs continuously, or runs on its own platform. + + A stream that declares ``execution.retries`` has a keepalive DAG, so it counts as scheduled. + ``keepalive=False`` is for a renderer that has no keepalive DAG (``scaffold-composer``). + """ builds = [b for b in get_builds(contract) if isinstance(b, dict)] if not builds: return False managed = _managed_lane(contract) return all( - _classify(b, managed)[0] == _OUTSIDE and not _keeps_alive(b, managed) for b in builds + _classify(b, managed)[0] == _OUTSIDE and not (keepalive and _keeps_alive(b, managed)) + for b in builds ) @@ -1058,12 +1065,20 @@ def _expose_ref_uri(self, where: str, ref: Mapping[str, Any]) -> str: def check_expose_refs(self, dags: List[ScheduledDag]) -> None: """Refuse a dataset trigger on an expose of this contract that no task updates: every - build that writes it runs outside any DAG, so the trigger would never fire.""" - tasked = {t.build_id for dag in dags for t in dag.tasks} + build that writes it runs outside any DAG, or only in a keepalive DAG, which declares no + outlets, so the trigger would never fire.""" + tasked = {t.build_id for dag in dags if not dag.keepalive for t in dag.tasks} for where, expose_id, writers in self.expose_refs: if not writers or any(_expose_writer_id(b) in tasked for b in writers): continue names = ", ".join(repr(b.get("id")) for b in writers) + if all(_keeps_alive(b, self.managed) for b in writers): + raise ScheduleRenderError( + f"{where}: expose {expose_id!r} is written by {names}, whose keepalive DAG " + "declares no outlets (a stream updates its tables all the time), so no task " + "ever updates its asset and the trigger would never fire. Run this build on " + "a schedule (trigger.type: schedule)" + ) raise ScheduleRenderError( f"{where}: expose {expose_id!r} is written by {names}, which no DAG runs " f"({self._why_no_task(writers[0])}), so no task ever updates its asset and the " diff --git a/tests/cli/test_generate_schedule_build_graph.py b/tests/cli/test_generate_schedule_build_graph.py index 7da62bfa..b2eb2830 100644 --- a/tests/cli/test_generate_schedule_build_graph.py +++ b/tests/cli/test_generate_schedule_build_graph.py @@ -21,6 +21,7 @@ import yaml from fluid_build.cli import generate_schedule +from fluid_build.schedulers.airflow import fluid_apply from tests.cli._schedule_dag_fixtures import StubAsset, load_dag, run_task from tests.schedulers.fluid_apply_fixtures import SPARK_CONTRACT, events_pipeline @@ -102,6 +103,30 @@ def test_a_contract_of_streams_alone_needs_no_dag( assert not (tmp_path / "out").exists() +def test_a_kept_alive_stream_gets_its_keepalive_dag( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + stream = dict(SPARK_CONTRACT["builds"][0], execution={"retries": {"maxAttempts": 5}}) + _project(tmp_path, dict(SPARK_CONTRACT, builds=[stream])) + assert _generate(tmp_path, monkeypatch) == 0 + assert sorted(p.name for p in (tmp_path / "out").iterdir()) == ["stream_raw_dag.py"] + + +@pytest.mark.parametrize("scheduler", ["prefect", "dagster"]) +def test_a_scheduler_with_no_keepalive_has_nothing_to_schedule_for_a_stream( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture, scheduler: str +) -> None: + """Only the Airflow DAGs keep a stream running: another scheduler writes nothing for it, + and is told where the keepalive DAG comes from.""" + stream = dict(SPARK_CONTRACT["builds"][0], execution={"retries": {"maxAttempts": 5}}) + _project(tmp_path, dict(SPARK_CONTRACT, builds=[stream])) + assert _generate(tmp_path, monkeypatch, "--scheduler", scheduler) == 0 + out = " ".join(capsys.readouterr().out.split()) + assert "No build needs a DAG" in out + assert "gets a keepalive DAG on Airflow: fluid generate schedule --scheduler airflow" in out + assert not (tmp_path / "out").exists() + + def test_dag_id_and_schedule_overrides_stay_refused_for_a_build_graph( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: @@ -128,6 +153,28 @@ def test_the_composer_scaffold_agrees_that_streams_need_no_dag( assert not (tmp_path / "dags").exists() +def test_the_composer_scaffold_leaves_a_kept_alive_stream_to_generate_schedule( + tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + """Its validate, plan and apply DAG cannot keep a stream running: it writes none, and says + which command writes the stream's keepalive DAG.""" + from types import SimpleNamespace + + from fluid_build.cli.scaffold_composer import run as scaffold_composer + + contract = tmp_path / "contract.fluid.yaml" + stream = dict(SPARK_CONTRACT["builds"][0], execution={"retries": {"maxAttempts": 5}}) + data = dict(SPARK_CONTRACT, builds=[stream]) + assert fluid_apply.has_scheduled_builds(data) # generate schedule writes its keepalive DAG + contract.write_text(yaml.safe_dump(data), "utf-8") + args = SimpleNamespace(contract=str(contract), env=None, out_dir=str(tmp_path / "dags")) + assert scaffold_composer(args, LOG) == 0 + out = " ".join(capsys.readouterr().out.split()) + assert "No DAG written." in out + assert "gets a keepalive DAG from fluid generate schedule" in out + assert not (tmp_path / "dags").exists() + + def test_regenerating_removes_a_dag_whose_build_now_runs_in_another( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture ) -> None: diff --git a/tests/schedulers/test_fluid_apply_keepalive.py b/tests/schedulers/test_fluid_apply_keepalive.py index 0091a2c0..f8b69d86 100644 --- a/tests/schedulers/test_fluid_apply_keepalive.py +++ b/tests/schedulers/test_fluid_apply_keepalive.py @@ -111,6 +111,29 @@ def test_fluid_apply_accepts_the_keepalive_argv(): _check_build_id_mode(args.build_id, mode) +def test_a_dataset_trigger_on_what_only_a_kept_alive_stream_writes_is_refused(): + """A keepalive DAG declares no outlets, so a trigger on an expose only the stream writes + would never fire: it is refused, as it is for a stream with no keepalive.""" + contract = _contract() + trigger = { + "type": "dataset", + "datasets": [{"productId": contract["id"], "exposeId": "events_raw"}], + } + contract["builds"].append( + {"id": "consumer", "engine": "dbt", "outputs": [], "execution": {"trigger": trigger}} + ) + with pytest.raises(fluid_apply.ScheduleRenderError) as exc: + _files(contract) + message = str(exc.value) + assert "is written by 'stream-raw', whose keepalive DAG declares no outlets" in message + assert "the trigger would never fire" in message + + +def test_a_stream_its_platform_keeps_running_gets_no_keepalive(): + """trigger.type: continuous puts a build outside every scheduler, retries or not.""" + assert sorted(_files(_contract(trigger={"type": "continuous"}))) == ["daily_curated_dag.py"] + + def test_without_retries_a_stream_still_gets_no_dag(): assert sorted(_files(copy.deepcopy(SPARK_CONTRACT))) == ["daily_curated_dag.py"] From 0c2f843f457a874908e0c13d84e0f8e2d63c1d5f Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 19 Nov 2026 02:31:26 +0100 Subject: [PATCH 05/10] docs(schedule): name every case a keepalive run fails on The keepalive DAG's text said a run fails when the stream drifted or the budget is spent. fluid apply --mode reconcile also fails when nothing is deployed or the stream's cluster ended, and on EMR Serverless, EMR on EKS and Glue the service's own retries are the restarts. It leaves a run cancelled on purpose stopped until fluid apply starts it. The DAG text, the KEEPALIVE_APPLY_ARGS comment and the CHANGELOG entry now say so. --- CHANGELOG.md | 3 ++- fluid_build/schedulers/airflow/fluid_apply.py | 19 +++++++++++-------- .../kept_alive_stream/stream_raw_dag.py.txt | 12 +++++++----- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da520624..540d9cc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -123,7 +123,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 interruption or a failed driver stopped starts again on its checkpoint. It runs every 5 minutes, or on the build's `execution.orchestration.airflow.dagConfig.schedule`, one run at a time, with no catch-up and no Airflow retries: the build's `execution.retries` is the restart - budget, and a run fails when the stream drifted from the contract or the budget is spent. The + budget, and a run fails when the stream cannot be kept running (it drifted from the contract, + nothing is deployed, or the restarts are spent). A run cancelled on purpose stays stopped. The task declares no outlets, so a dataset trigger on what only the stream writes is refused, as before. A contract whose only builds are such streams now gets this DAG on Airflow instead of "No build needs a DAG"; on Prefect or Dagster, and in `fluid scaffold-composer`, it still has diff --git a/fluid_build/schedulers/airflow/fluid_apply.py b/fluid_build/schedulers/airflow/fluid_apply.py index 2336f744..37bb74fc 100644 --- a/fluid_build/schedulers/airflow/fluid_apply.py +++ b/fluid_build/schedulers/airflow/fluid_apply.py @@ -126,9 +126,10 @@ #: What a keepalive DAG passes to ``fluid apply`` after the contract and ``--env``, before #: ``--build-id --yes``. ``--mode reconcile`` runs no ``tofu apply``: it starts that one -#: stream again on its checkpoint when it stopped, within the restart budget the build's -#: ``execution.retries`` sets, and exits non-zero when the stream drifted from the contract or -#: the budget is spent. The one place a keepalive's apply arguments are named. +#: stream again on its checkpoint when it died, within the restart budget the build's +#: ``execution.retries`` sets, and leaves a run cancelled on purpose stopped. It exits non-zero +#: when it cannot keep the stream running (drift from the contract, nothing deployed, the +#: restarts spent). The one place a keepalive's apply arguments are named. KEEPALIVE_APPLY_ARGS: Tuple[str, ...] = ("--mode", "reconcile") #: How often a keepalive DAG reconciles its stream, unless the build's @@ -1440,11 +1441,13 @@ def _wrap(words: List[str], indent: str, width: int = 76) -> str: ' fluid apply "$FLUID_PROJECT_DIR/$CONTRACT_PATH" --env "$FLUID_ENV_NAME" \\\\\n' " " + _apply_args(KEEPALIVE_APPLY_ARGS, '"$BUILD_ID"') + "\n" "\n" - "It runs no tofu apply. It starts the stream again on its checkpoint when it\n" - "stopped (a Spot interruption, a failed driver), within the restart budget the\n" - "build's execution.retries sets, and fails the run when the stream drifted\n" - "from the contract or the budget is spent, so the task itself never retries.\n" - "It runs on SCHEDULE, one run at a time, with no catch-up. The build's\n" + "It runs no tofu apply. When the stream died (a Spot interruption, a failed\n" + "driver), it starts again on its checkpoint, within the restart budget the\n" + "build's execution.retries sets, so the task itself never retries. The run\n" + "fails when the stream cannot be kept running: it drifted from the contract,\n" + "nothing is deployed, its cluster ended, or the restarts are spent. A run\n" + "cancelled on purpose stays stopped until fluid apply starts it. It runs on\n" + "SCHEDULE, one run at a time, with no catch-up. The build's\n" "execution.orchestration.airflow.dagConfig.schedule sets SCHEDULE.\n" ) diff --git a/tests/schedulers/golden/fluid_apply/kept_alive_stream/stream_raw_dag.py.txt b/tests/schedulers/golden/fluid_apply/kept_alive_stream/stream_raw_dag.py.txt index f225c5fe..4ea4e45b 100644 --- a/tests/schedulers/golden/fluid_apply/kept_alive_stream/stream_raw_dag.py.txt +++ b/tests/schedulers/golden/fluid_apply/kept_alive_stream/stream_raw_dag.py.txt @@ -7,11 +7,13 @@ constants below filled in (`--env` is left out when FLUID_ENV_NAME is empty): fluid apply "$FLUID_PROJECT_DIR/$CONTRACT_PATH" --env "$FLUID_ENV_NAME" \\ --mode reconcile --build-id "$BUILD_ID" --yes -It runs no tofu apply. It starts the stream again on its checkpoint when it -stopped (a Spot interruption, a failed driver), within the restart budget the -build's execution.retries sets, and fails the run when the stream drifted -from the contract or the budget is spent, so the task itself never retries. -It runs on SCHEDULE, one run at a time, with no catch-up. The build's +It runs no tofu apply. When the stream died (a Spot interruption, a failed +driver), it starts again on its checkpoint, within the restart budget the +build's execution.retries sets, so the task itself never retries. The run +fails when the stream cannot be kept running: it drifted from the contract, +nothing is deployed, its cluster ended, or the restarts are spent. A run +cancelled on purpose stays stopped until fluid apply starts it. It runs on +SCHEDULE, one run at a time, with no catch-up. The build's execution.orchestration.airflow.dagConfig.schedule sets SCHEDULE. Worker requirements: From 3ab0f76c708ede2348a74a8efeebd750db26d817 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 19 Nov 2026 06:06:14 +0100 Subject: [PATCH 06/10] test(schedulers): give each registry test its own scheduler registry The autouse fixture in test_scheduler_framework.py emptied the shared scheduler registry and left it empty. The built-in schedulers register only when fluid_build.schedulers is first imported, so a later test in the same worker that looked up prefect or dagster by name found no scheduler. CI runs the suite under xdist with pytest-randomly, so such a test failed only on some orders. Each test now gets an empty registry of its own through monkeypatch, and the shared one is back in place after it. --- tests/test_scheduler_framework.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/test_scheduler_framework.py b/tests/test_scheduler_framework.py index 2e7320fd..4073291e 100644 --- a/tests/test_scheduler_framework.py +++ b/tests/test_scheduler_framework.py @@ -12,6 +12,7 @@ import pytest from fluid_build.engines.base import Severity, ValidationIssue +from fluid_build.schedulers import registry from fluid_build.schedulers.base import ScheduleEngine, ScheduleGenerationResult, ScheduleIntent from fluid_build.schedulers.registry import ( _reset_registry, @@ -28,11 +29,14 @@ @pytest.fixture(autouse=True) -def _clean_registry(): - """Reset the scheduler registry before and after each test.""" - _reset_registry() - yield - _reset_registry() +def _clean_registry(monkeypatch: pytest.MonkeyPatch) -> None: + """Give each test an empty scheduler registry, and put the shared one back after it. + + The built-in schedulers register once, when ``fluid_build.schedulers`` is first + imported. Emptying the shared registry would hide them from every later test in the + same process, and the test order is random. + """ + monkeypatch.setattr(registry, "_SCHEDULERS", {}) class _DummyScheduler(ScheduleEngine): From ee1cb716abfc2c1ca9db706b02d46efa4b233714 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 19 Nov 2026 07:16:27 +0100 Subject: [PATCH 07/10] docs(schedule): name the job-config and worker-settings cases a keepalive fails on The keepalive DAG text and its CHANGELOG entry now also name two cases a reconcile exits 1 on. The job config the last apply deployed cannot be read: it is read on every run, while the stream is healthy too. The worker's FLUID_SPARK_* settings or forge release differ from the CI apply's: a policy gate refuses the build, or the compiled config reads as drift. --- CHANGELOG.md | 4 +++- fluid_build/schedulers/airflow/fluid_apply.py | 8 +++++--- .../fluid_apply/kept_alive_stream/stream_raw_dag.py.txt | 8 +++++--- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea376d96..3caf5a1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -344,7 +344,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 minutes, or on the build's `execution.orchestration.airflow.dagConfig.schedule`, one run at a time, with no catch-up and no Airflow retries: the build's `execution.retries` is the restart budget, and a run fails when the stream cannot be kept running (it drifted from the contract, - nothing is deployed, or the restarts are spent). A run cancelled on purpose stays stopped. The + nothing is deployed, its job config cannot be read, its cluster ended, or the restarts are + spent). The worker needs the `FLUID_SPARK_*` settings and the forge release the CI apply has, + or a run can fail too. A run cancelled on purpose stays stopped. The task declares no outlets, so a dataset trigger on what only the stream writes is refused, as before. A contract whose only builds are such streams now gets this DAG on Airflow instead of "No build needs a DAG"; on Prefect or Dagster, and in `fluid scaffold-composer`, it still has diff --git a/fluid_build/schedulers/airflow/fluid_apply.py b/fluid_build/schedulers/airflow/fluid_apply.py index 37bb74fc..5cbdbff0 100644 --- a/fluid_build/schedulers/airflow/fluid_apply.py +++ b/fluid_build/schedulers/airflow/fluid_apply.py @@ -1445,9 +1445,11 @@ def _wrap(words: List[str], indent: str, width: int = 76) -> str: "driver), it starts again on its checkpoint, within the restart budget the\n" "build's execution.retries sets, so the task itself never retries. The run\n" "fails when the stream cannot be kept running: it drifted from the contract,\n" - "nothing is deployed, its cluster ended, or the restarts are spent. A run\n" - "cancelled on purpose stays stopped until fluid apply starts it. It runs on\n" - "SCHEDULE, one run at a time, with no catch-up. The build's\n" + "nothing is deployed, the job config the last apply deployed cannot be read,\n" + "its cluster ended, or the restarts are spent. It can also fail when this\n" + "worker's FLUID_SPARK_* settings or forge release differ from the CI apply's.\n" + "A run cancelled on purpose stays stopped until fluid apply starts it. It\n" + "runs on SCHEDULE, one run at a time, with no catch-up. The build's\n" "execution.orchestration.airflow.dagConfig.schedule sets SCHEDULE.\n" ) diff --git a/tests/schedulers/golden/fluid_apply/kept_alive_stream/stream_raw_dag.py.txt b/tests/schedulers/golden/fluid_apply/kept_alive_stream/stream_raw_dag.py.txt index 4ea4e45b..ff0cd8c2 100644 --- a/tests/schedulers/golden/fluid_apply/kept_alive_stream/stream_raw_dag.py.txt +++ b/tests/schedulers/golden/fluid_apply/kept_alive_stream/stream_raw_dag.py.txt @@ -11,9 +11,11 @@ It runs no tofu apply. When the stream died (a Spot interruption, a failed driver), it starts again on its checkpoint, within the restart budget the build's execution.retries sets, so the task itself never retries. The run fails when the stream cannot be kept running: it drifted from the contract, -nothing is deployed, its cluster ended, or the restarts are spent. A run -cancelled on purpose stays stopped until fluid apply starts it. It runs on -SCHEDULE, one run at a time, with no catch-up. The build's +nothing is deployed, the job config the last apply deployed cannot be read, +its cluster ended, or the restarts are spent. It can also fail when this +worker's FLUID_SPARK_* settings or forge release differ from the CI apply's. +A run cancelled on purpose stays stopped until fluid apply starts it. It +runs on SCHEDULE, one run at a time, with no catch-up. The build's execution.orchestration.airflow.dagConfig.schedule sets SCHEDULE. Worker requirements: From 8325408c2d3ce55171fd4823303cccd658170556 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 19 Nov 2026 08:39:35 +0100 Subject: [PATCH 08/10] docs(changelog): list the keepalive DAG under Unreleased The 0.33.0 cut turned the old Unreleased section into 0.33.0, and this entry moved with it. The keepalive DAG is not in 0.33.0, so it is listed under Unreleased, after the entry for Spark stream batch bounds. --- CHANGELOG.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66d077bc..1c3be73c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `fluid.dead-letter.raw-key` header: the Kafka sink is at least once, and a batch sent again after a failure carries the same keys, so a consumer drops repeats by key. +- **A keepalive DAG for an always-on stream that declares `execution.retries`.** `fluid + generate schedule` gives each forge Spark stream that is not `trigger.availableNow` and + declares `execution.retries` a small DAG whose one task runs + `fluid apply --mode reconcile --build-id --yes`, so a stream a Spot + interruption or a failed driver stopped starts again on its checkpoint. It runs every 5 + minutes, or on the build's `execution.orchestration.airflow.dagConfig.schedule`, one run at a + time, with no catch-up and no Airflow retries: the build's `execution.retries` is the restart + budget, and a run fails when the stream cannot be kept running (it drifted from the contract, + nothing is deployed, its job config cannot be read, its cluster ended, or the restarts are + spent). The worker needs the `FLUID_SPARK_*` settings and the forge release the CI apply has, + or a run can fail too. A run cancelled on purpose stays stopped. The task declares no + outlets, so a dataset trigger on what only the stream writes is refused, as before. A contract + whose only builds are such streams now gets this DAG on Airflow instead of "No build needs a + DAG"; on Prefect or Dagster, and in `fluid scaffold-composer`, it still has nothing to + schedule, and the message names the Airflow keepalive. + ### Fixed - **`fluid apply` runs a dbt build whatever its trigger, finds the dbt that has its adapter, and @@ -456,22 +472,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 with the type and mode apply wrote, and `fluid diff` reads a map as it reads an array and every spelling of a struct (`struct`, `record`, `row`) as the RECORD BigQuery reports. -- **A keepalive DAG for an always-on stream that declares `execution.retries`.** `fluid - generate schedule` gives each forge Spark stream that is not `trigger.availableNow` and - declares `execution.retries` a small DAG whose one task runs - `fluid apply --mode reconcile --build-id --yes`, so a stream a Spot - interruption or a failed driver stopped starts again on its checkpoint. It runs every 5 - minutes, or on the build's `execution.orchestration.airflow.dagConfig.schedule`, one run at a - time, with no catch-up and no Airflow retries: the build's `execution.retries` is the restart - budget, and a run fails when the stream cannot be kept running (it drifted from the contract, - nothing is deployed, its job config cannot be read, its cluster ended, or the restarts are - spent). The worker needs the `FLUID_SPARK_*` settings and the forge release the CI apply has, - or a run can fail too. A run cancelled on purpose stays stopped. The - task declares no outlets, so a dataset trigger on what only the stream writes is refused, as - before. A contract whose only builds are such streams now gets this DAG on Airflow instead of - "No build needs a DAG"; on Prefect or Dagster, and in `fluid scaffold-composer`, it still has - nothing to schedule, and the message names the Airflow keepalive. - - **`fluid generate schedule` renders the build graph: `builds[].dependencies` and dataset triggers.** Builds that `builds[].dependencies` link now run in one Airflow DAG, one `fluid apply --build-id` task per build, each after the builds it names. A dependency's From 2207c0d72c540daebfc590b00bea6dd06ef7e7cf Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 19 Nov 2026 08:51:38 +0100 Subject: [PATCH 09/10] docs(agents): say that generate schedule renders a stream's keepalive DAG The reconcile paragraph names a keepalive as a scheduler task every few minutes. fluid generate schedule now renders one, as an Airflow DAG, for a stream that declares execution.retries. --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 79917f13..25a0f05c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,7 +49,7 @@ Every contract is delivered through an 11-stage pipeline. Each stage is a hard g | `replace` | auto-snapshot → `CREATE OR REPLACE TABLE` | — | **dropped**; backup retained | | `replace-and-build` | same as `replace` | `dbt run --full-refresh` | **dropped**; rebuilt | -`--mode reconcile [--build-id ]` runs no `tofu`, no DDL and no build, and refuses a `plan.json`: it keeps the contract's always-on Spark streams running with the config the last apply deployed (the job config it uploaded, and the config hash its runs carry). A stream that died starts again on its checkpoint within `execution.retries.maxAttempts` restarts per rolling hour (5 by default, at most 10); drift, a spent budget or nothing deployed starts nothing and exits 1; a run cancelled through the runtime is left stopped. A keepalive (a scheduler task every few minutes, with `--build-id `) keeps a stream up. A plain `fluid apply` restarts a dead stream under the same budget. EMR Serverless and EMR on EKS retry failed runs themselves, so forge never restarts a FAILED run there; Glue's job is emitted with MaxRetries 0, so forge restarts Glue streams as it does EMR on EC2 and local ones. Code: `build_runners/spark/_reconcile.py`, `cli/_spark_apply.py::reconcile_only`. +`--mode reconcile [--build-id ]` runs no `tofu`, no DDL and no build, and refuses a `plan.json`: it keeps the contract's always-on Spark streams running with the config the last apply deployed (the job config it uploaded, and the config hash its runs carry). A stream that died starts again on its checkpoint within `execution.retries.maxAttempts` restarts per rolling hour (5 by default, at most 10); drift, a spent budget or nothing deployed starts nothing and exits 1; a run cancelled through the runtime is left stopped. A keepalive (a scheduler task every few minutes, with `--build-id `) keeps a stream up; `fluid generate schedule` renders one as an Airflow DAG for a stream that declares `execution.retries`. A plain `fluid apply` restarts a dead stream under the same budget. EMR Serverless and EMR on EKS retry failed runs themselves, so forge never restarts a FAILED run there; Glue's job is emitted with MaxRetries 0, so forge restarts Glue streams as it does EMR on EC2 and local ones. Code: `build_runners/spark/_reconcile.py`, `cli/_spark_apply.py::reconcile_only`. `--allow-data-loss` is required for `replace*` when `FLUID_ENV != dev` OR target has rows. `--no-verify-plan-binding` is an emergency DR waiver for plan-binding checks (logged at WARNING level so audit trails catch it). From 3e99b8d24bfe12c883be4df0cc181f375fbfc4eb Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 19 Nov 2026 08:51:38 +0100 Subject: [PATCH 10/10] test(schedule): import each test helper once The build-graph tests import scaffold_composer, SimpleNamespace and fluid_apply at module level instead of again inside each test, and the registry fixture patches _SCHEDULERS by its dotted path, so the module needs no second import of the registry. --- tests/cli/test_generate_schedule_build_graph.py | 12 ++---------- tests/test_scheduler_framework.py | 3 +-- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/tests/cli/test_generate_schedule_build_graph.py b/tests/cli/test_generate_schedule_build_graph.py index b2eb2830..bdb0081a 100644 --- a/tests/cli/test_generate_schedule_build_graph.py +++ b/tests/cli/test_generate_schedule_build_graph.py @@ -16,11 +16,13 @@ import logging import os from pathlib import Path +from types import SimpleNamespace import pytest import yaml from fluid_build.cli import generate_schedule +from fluid_build.cli.scaffold_composer import run as scaffold_composer from fluid_build.schedulers.airflow import fluid_apply from tests.cli._schedule_dag_fixtures import StubAsset, load_dag, run_task from tests.schedulers.fluid_apply_fixtures import SPARK_CONTRACT, events_pipeline @@ -139,10 +141,6 @@ def test_dag_id_and_schedule_overrides_stay_refused_for_a_build_graph( def test_the_composer_scaffold_agrees_that_streams_need_no_dag( tmp_path: Path, capsys: pytest.CaptureFixture ) -> None: - from types import SimpleNamespace - - from fluid_build.cli.scaffold_composer import run as scaffold_composer - contract = tmp_path / "contract.fluid.yaml" stream = dict(SPARK_CONTRACT["builds"][0], execution={"trigger": {"cron": "@daily"}}) contract.write_text(yaml.safe_dump(dict(SPARK_CONTRACT, builds=[stream])), "utf-8") @@ -158,10 +156,6 @@ def test_the_composer_scaffold_leaves_a_kept_alive_stream_to_generate_schedule( ) -> None: """Its validate, plan and apply DAG cannot keep a stream running: it writes none, and says which command writes the stream's keepalive DAG.""" - from types import SimpleNamespace - - from fluid_build.cli.scaffold_composer import run as scaffold_composer - contract = tmp_path / "contract.fluid.yaml" stream = dict(SPARK_CONTRACT["builds"][0], execution={"retries": {"maxAttempts": 5}}) data = dict(SPARK_CONTRACT, builds=[stream]) @@ -187,8 +181,6 @@ def test_regenerating_removes_a_dag_whose_build_now_runs_in_another( assert _generate(tmp_path, monkeypatch) == 0 out = tmp_path / "out" assert (out / "daily_users_dag.py").exists() - from fluid_build.schedulers.airflow import fluid_apply - rendered = fluid_apply.render_fluid_apply_dags(contract, env="aws", contract_path=CONTRACT) other_env = rendered["daily_users_dag.py"] (out / "daily_users_other_env_dag.py").write_text(other_env, encoding="utf-8") diff --git a/tests/test_scheduler_framework.py b/tests/test_scheduler_framework.py index 4073291e..aec19107 100644 --- a/tests/test_scheduler_framework.py +++ b/tests/test_scheduler_framework.py @@ -12,7 +12,6 @@ import pytest from fluid_build.engines.base import Severity, ValidationIssue -from fluid_build.schedulers import registry from fluid_build.schedulers.base import ScheduleEngine, ScheduleGenerationResult, ScheduleIntent from fluid_build.schedulers.registry import ( _reset_registry, @@ -36,7 +35,7 @@ def _clean_registry(monkeypatch: pytest.MonkeyPatch) -> None: imported. Emptying the shared registry would hide them from every later test in the same process, and the test order is random. """ - monkeypatch.setattr(registry, "_SCHEDULERS", {}) + monkeypatch.setattr("fluid_build.schedulers.registry._SCHEDULERS", {}) class _DummyScheduler(ScheduleEngine):