From 376fd4c5f13ebf8fcd826b5dbcce4c0d9aa50544 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 19 Nov 2026 08:57:43 +0100 Subject: [PATCH 1/2] fix(spark): let the scheduler decrypt the artifacts a start reads back A start reads the job config and the runner back from S3 before it runs (checked_run), as the scheduler's identity for a scheduled build. Its policy granted s3:GetObject on them and no kms:Decrypt, so with an SSE-KMS artifact bucket every scheduled start failed with AccessDenied. The caller policy now carries an S3Kms statement like the job role's: kms:Decrypt on any key, only through S3 in the provider's region (kms:ViaService) and only for those objects, or their bucket under an S3 Bucket Key (kms:EncryptionContext:aws:s3:arn). The largest scheduler policy, an isolated EMR Serverless build, is 1,766 characters of IAM's 6,144. --- fluid_build/iac/providers/aws_identity.py | 40 ++++++++++++++++- fluid_build/iac/providers/aws_ingest.py | 1 + tests/iac/test_iac_aws_spark_infra.py | 54 +++++++++++++++++++++++ 3 files changed, 94 insertions(+), 1 deletion(-) diff --git a/fluid_build/iac/providers/aws_identity.py b/fluid_build/iac/providers/aws_identity.py index 6a17081f..71d2b51c 100644 --- a/fluid_build/iac/providers/aws_identity.py +++ b/fluid_build/iac/providers/aws_identity.py @@ -1032,6 +1032,9 @@ class CallerRun: objects: Tuple[Any, ...] #: An isolated EMR Serverless application is found by name: ListApplications. lookup_application: bool = False + #: The ARN of the bucket ``objects`` are in: under an S3 Bucket Key, the encryption + #: context of their keys. + bucket: Any = None def _caller_targets(run: CallerRun, *, account: Any, region: Any) -> List[Any]: @@ -1062,7 +1065,11 @@ def caller_document(runs: Sequence[CallerRun], *, account: Any, region: Any) -> One statement per runtime (its actions once, on every target of the product's builds on that runtime), one PassRole per runtime, and one for the artifacts: a managed policy holds at most 6,144 characters, and a statement per build reached it at about six - builds. Each build's role and target is still named, so nothing else is granted.""" + builds. Each build's role and target is still named, so nothing else is granted. + + A start reads the job config and the runner back (``checked_run`` in + build_runners/spark/_adapters.py), so the artifacts get ``s3:GetObject`` and, through S3 + only, ``kms:Decrypt``: an SSE-KMS artifact bucket would refuse the read without it.""" statements: List[Dict[str, Any]] = [] runtimes = _unique([run.runtime for run in runs]) for runtime in runtimes: @@ -1126,6 +1133,37 @@ def caller_document(runs: Sequence[CallerRun], *, account: Any, region: Any) -> "resources": _unique([obj for run in runs for obj in run.objects]), } ) + # With an S3 Bucket Key the encryption context is the bucket's ARN; without one it is + # the object's. + contexts = _unique( + [ + context + for run in runs + for context in [*([run.bucket] if run.bucket is not None else []), *run.objects] + ] + ) + if contexts: + statements.append( + { + # As the job role's own S3Kms: whatever key the bucket encrypts them with. + "sid": "S3Kms", + "effect": "Allow", + "actions": ["kms:Decrypt"], + "resources": ["*"], + "condition": [ + { + "test": "StringEquals", + "variable": "kms:ViaService", + "values": [_join("s3.", region, ".amazonaws.com")], + }, + { + "test": "StringLike", + "variable": "kms:EncryptionContext:aws:s3:arn", + "values": contexts, + }, + ], + } + ) return {"statement": statements} diff --git a/fluid_build/iac/providers/aws_ingest.py b/fluid_build/iac/providers/aws_ingest.py index 1bf0f9bb..0835c56e 100644 --- a/fluid_build/iac/providers/aws_ingest.py +++ b/fluid_build/iac/providers/aws_ingest.py @@ -527,6 +527,7 @@ def _emit_build( aws_identity._join(bucket_arn, "/", b.runner_key), ), lookup_application=b.runtime == "emr-serverless" and not b.cfg.get("applicationId"), + bucket=bucket_arn, ) diff --git a/tests/iac/test_iac_aws_spark_infra.py b/tests/iac/test_iac_aws_spark_infra.py index ff8e0e05..d2a9b230 100644 --- a/tests/iac/test_iac_aws_spark_infra.py +++ b/tests/iac/test_iac_aws_spark_infra.py @@ -829,6 +829,58 @@ def test_a_run_to_completion_build_gets_the_schedulers_policy(contract): assert not any(s.startswith("FindApplication") for s in sids) +def test_the_scheduler_may_decrypt_the_artifacts_it_reads_back_through_s3_only(contract): + """A start reads job.json and the runner back (``_adapters.checked_run``); an SSE-KMS + artifact bucket refuses that read without ``kms:Decrypt``.""" + contract["builds"][0]["properties"]["stream"]["trigger"] = {"availableNow": True} + _, data = emit(contract) + document = data["aws_iam_policy_document"]["commerce_orders_stream_spark_caller"] + kms = statement(document, "S3Kms") + assert kms["actions"] == ["kms:Decrypt"] + assert kms["resources"] == ["*"] + via, context = kms["condition"] + assert (via["test"], via["variable"]) == ("StringEquals", "kms:ViaService") + assert via["values"] == [ + f"s3.${{data.aws_region.{aws_ingest.REGION_DATA}.region}}.amazonaws.com" + ] + assert (context["test"], context["variable"]) == ( + "StringLike", + "kms:EncryptionContext:aws:s3:arn", + ) + artifacts = next(s for s in document["statement"] if s["sid"].startswith("Artifacts")) + # Under an S3 Bucket Key the context is the bucket's ARN; without one, the object's. + assert context["values"] == ["arn:aws:s3:::commerce-lake", *artifacts["resources"]] + + +def _iam_policy_chars(document): + """The policy as IAM counts it: its JSON without whitespace (OpenTofu expressions are + counted as written, longer than the account ids and regions they resolve to).""" + statements = [] + for s in document["statement"]: + rendered = { + "Sid": s["sid"], + "Effect": s["effect"], + "Action": s["actions"], + "Resource": s["resources"], + } + for c in s.get("condition") or []: + rendered.setdefault("Condition", {}).setdefault(c["test"], {})[c["variable"]] = c[ + "values" + ] + statements.append(rendered) + policy = {"Version": "2012-10-17", "Statement": statements} + return len(json.dumps(policy, separators=(",", ":"))) + + +@pytest.mark.parametrize("runtime", ["emr-serverless", "glue", "emr-eks", "emr-ec2"]) +def test_the_schedulers_policy_fits_a_managed_policy_on_every_runtime(contract, runtime): + contract["builds"][0]["properties"]["stream"]["trigger"] = {"availableNow": True} + _, data = emit(_runtime(contract, runtime)) + document = data["aws_iam_policy_document"]["commerce_orders_stream_spark_caller"] + assert statement(document, "S3Kms") + assert _iam_policy_chars(document) < 6144 # IAM's limit for a managed policy + + # --- Lake Formation writes (item 4) --------------------------------------------------- @@ -1196,12 +1248,14 @@ def test_the_schedulers_policy_names_each_action_once_per_runtime(): "RunEmrEc2", "StepRoleEmrEc2", "Artifacts", + "S3Kms", ] by_sid = {s["sid"]: s for s in doc["statement"]} serverless = by_sid["RunEmrServerless"]["resources"] assert len(serverless) == 4 # two applications, each named once with its job runs assert len(by_sid["PassRoleEmrServerless"]["resources"]) == 8 assert len(by_sid["Artifacts"]["resources"]) == 9 + assert by_sid["S3Kms"]["condition"][1]["values"] == by_sid["Artifacts"]["resources"] assert len(json.dumps(doc, separators=(",", ":"))) < 6144 From 92da37083f52c86850bf607a0869cacc11a006a6 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 19 Nov 2026 08:57:43 +0100 Subject: [PATCH 2/2] fix(spark): check the deployed config before cancelling a stream's old run fluid apply cancelled a stream's run of another config, waited for it to stop, and only then started the new one, whose start reads the deployed job config and runner back and refuses them when another apply has changed them. A refused start so left the stream stopped until the next apply. When a run of another config is to be cancelled, the reconcile now reads the two artifacts back first (Adapter.checked, read once: the start then uses the run it returned). A refusal is the build's refused outcome naming the run left running, nothing is cancelled or started, and fluid apply exits 1. A start with nothing to cancel keeps its order (on EMR on EC2 the policy check and the ended-cluster check still share one DescribeCluster). --- CHANGELOG.md | 11 ++++++ fluid_build/build_runners/spark/_adapters.py | 13 ++++++- fluid_build/build_runners/spark/_reconcile.py | 20 +++++++++- tests/build_runners/spark/test_adapters.py | 7 ++++ tests/build_runners/spark/test_reconcile.py | 39 ++++++++++++++++++- 5 files changed, 86 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e1fdb5a..dd6cb5b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -135,6 +135,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `emr-serverless:StopApplication`, which the identity running the destroy needs beyond the runtime's list and cancel actions. +- **A scheduled Spark start reads its job config from an SSE-KMS bucket, and a start that is + refused leaves a stream's old run going.** A start reads `job.json` and the runner back from + S3 before it runs, as the scheduler's identity for a scheduled build, and the scheduler's + policy granted `s3:GetObject` on them and no `kms:Decrypt`: with a customer-managed key as the + artifact bucket's default encryption, every scheduled start failed with AccessDenied. The + policy now grants `kms:Decrypt` through S3 only (`kms:ViaService`), for those two objects, or + their bucket under an S3 Bucket Key. `fluid apply` also cancelled a stream's run of another + config before that read, so a start the deployed artifacts refused left the stream stopped + until the next apply. It now reads them first: a refusal cancels nothing, the old run keeps + going, and `fluid apply` exits 1. + ### Changed - **A Spark run starts only on the job config `fluid apply` compiled.** Every runtime, local diff --git a/fluid_build/build_runners/spark/_adapters.py b/fluid_build/build_runners/spark/_adapters.py index 2b5d69e2..afc8c664 100644 --- a/fluid_build/build_runners/spark/_adapters.py +++ b/fluid_build/build_runners/spark/_adapters.py @@ -265,10 +265,15 @@ def __init__(self, run: DesiredRun, clients: Optional[ClientFactory]) -> None: self.clients = clients # A local run has no AWS client: its "client" is the machine (``Local.host``). self.client = clients(self.service, run.region) if clients is not None else None + self._checked: Optional[DesiredRun] = None def checked(self) -> DesiredRun: - """:func:`checked_run` of this adapter's run.""" - return checked_run(self.run, self.clients) + """:func:`checked_run` of this adapter's run, read once: a reconcile checks before it + cancels the run of another config (a refusal leaves that run going), and the start + then uses the run it checked.""" + if self._checked is None: + self._checked = checked_run(self.run, self.clients) + return self._checked def runs(self, since: Optional[float] = None) -> List[RunInfo]: """The build's runs, newest first: every active one and the most recent ended ones, @@ -872,6 +877,10 @@ def runs(self, since: Optional[float] = None) -> List[RunInfo]: def get(self, run_id: str) -> RunInfo: return self._runs.get(run_id) + def checked(self) -> DesiredRun: + """Nothing is read back: each start writes this run's artifacts itself.""" + return self.run + def start(self, token: str, restart_of: Optional[str] = None) -> str: from fluid_build.build_runners.spark._local import start_run from fluid_build.spark import runner_source diff --git a/fluid_build/build_runners/spark/_reconcile.py b/fluid_build/build_runners/spark/_reconcile.py index c9e48a03..9e18be6b 100644 --- a/fluid_build/build_runners/spark/_reconcile.py +++ b/fluid_build/build_runners/spark/_reconcile.py @@ -13,7 +13,8 @@ one on the SAME checkpoint, so it resumes where the old one stopped. Spark's checkpoint holds the offsets of one topic on one cluster, so a new source on the old checkpoint is refused before anything is cancelled; moving ``properties.stream.checkpoint`` to a new - prefix is how a stream changes source. + prefix is how a stream changes source. A start the deployed job config or runner would + refuse (``_adapters.checked_run``) is refused before anything is cancelled too. 4. No running job: start one. When the newest run of this config ended, how it ended decides (:func:`_after_end`): @@ -480,6 +481,8 @@ def _reconcile_one( if only and confirm is not None and not confirm(_question(run, out, stale, matching, start)): out.action, out.message = "declined", _declined_text(stale, start) return out + if stale and not matching and not _checked_first(adapter, stale, out): + return out if not _cancel_and_wait(adapter, stale, out, clock, budget): return out if matching: # the new config already runs; the stale runs were extra @@ -552,6 +555,21 @@ def _question( return f"Spark {run.build} ({run.runtime}) on {out.where}: {what}?" +def _checked_first(adapter: Adapter, stale: Sequence[RunInfo], out: Outcome) -> bool: + """Read back the job config and runner the new run will start on + (:meth:`Adapter.checked`) before ``stale``, the runs of another config, are cancelled; + False (``out`` refused) when another apply has changed them: the old runs keep going.""" + try: + adapter.checked() + except SparkRunError as exc: + ids = ", ".join(s.run_id for s in stale) + they = f"run {ids} was" if len(stale) == 1 else f"runs {ids} were" + out.action, out.error = "refused", True + out.message = f"{_clean(exc)}; {they} left running" + return False + return True + + def _cancel_and_wait( adapter: Adapter, stale: Sequence[RunInfo], out: Outcome, clock: _Clock, budget: float ) -> bool: diff --git a/tests/build_runners/spark/test_adapters.py b/tests/build_runners/spark/test_adapters.py index 586f86e7..d19a257d 100644 --- a/tests/build_runners/spark/test_adapters.py +++ b/tests/build_runners/spark/test_adapters.py @@ -800,6 +800,13 @@ def test_a_config_this_apply_did_not_compile_starts_nothing(serverless, body): stub.assert_no_pending_responses() # nothing was sent to EMR +def test_the_check_before_a_cancel_and_the_start_read_the_artifacts_once(serverless): + adapter, _ = adapter_on(serverless, "emr-serverless") + bucket = adapter.clients("s3", serverless.region) + assert adapter.checked() is adapter.checked() + assert bucket.read == [serverless.job_uri, serverless.runner_uri] + + def test_a_runner_this_forge_does_not_ship_starts_nothing(serverless): adapter, stub = adapter_on(serverless, "emr-serverless", runner=b"import os # not ours\n") with pytest.raises(ad.SparkRunError, match="or its runner is not the one"): diff --git a/tests/build_runners/spark/test_reconcile.py b/tests/build_runners/spark/test_reconcile.py index e1c6401d..878590f9 100644 --- a/tests/build_runners/spark/test_reconcile.py +++ b/tests/build_runners/spark/test_reconcile.py @@ -6,6 +6,8 @@ from __future__ import annotations import dataclasses +import io +import logging import pytest @@ -13,7 +15,8 @@ ClientError = pytest.importorskip("botocore.exceptions").ClientError from fluid_build.build_runners.spark import _reconcile as rc -from fluid_build.build_runners.spark._adapters import RunInfo, SparkRunError +from fluid_build.build_runners.spark._adapters import RunInfo, SparkRunError, checked_run +from fluid_build.cli import _spark_apply ENV = {rc.WAIT_ENV: "5"} @@ -70,6 +73,9 @@ def __init__( def where(self): return "fake cluster" + def checked(self): + return self.run + def runs(self, since=None): self.since.append(since) if self.later is not None and "start" in self.calls: @@ -143,6 +149,37 @@ def test_another_config_is_cancelled_waited_for_and_replaced(run, clock, use): assert fake.calls == ["cancel old1", "start"] +class DeployedConfig: + """The artifact bucket after another apply: its job.json is not the one this apply + compiled.""" + + def get_object(self, Bucket, Key): # noqa: N803 - boto3's names + return {"Body": io.BytesIO(b'{"mode": "stream", "another": "config"}')} + + +def refusing(fake): + """``fake`` with the adapters' real check of the deployed artifacts.""" + fake.checked = lambda: checked_run(fake.run, lambda service, region: DeployedConfig()) + return fake + + +def test_a_start_the_deployed_config_refuses_leaves_the_old_run_going(run, clock, use): + fake = refusing(FakeAdapter([info("old1", "RUNNING", hash_="stale")])) + use(fake) + out = go(run, clock) + assert (out.action, out.error, out.cancelled, fake.calls) == ("refused", True, [], []) + assert "is not the one this apply compiled" in out.message + assert out.message.endswith("; run old1 was left running") + + +def test_fluid_apply_fails_when_the_deployed_config_refuses_the_start(run, use): + fake = refusing(FakeAdapter([info("old1", "RUNNING", hash_="stale")])) + use(fake) + log = logging.getLogger("test.spark.refused") + assert _spark_apply.reconcile_after_apply([run], log, dry_run=False) == 1 + assert fake.calls == [] # nothing cancelled, nothing started + + def test_a_new_topic_on_the_same_checkpoint_is_refused_before_anything_is_cancelled( run, clock, use ):