diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e1fdb5a..dd6cb5b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -135,6 +135,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `emr-serverless:StopApplication`, which the identity running the destroy needs beyond the runtime's list and cancel actions. +- **A scheduled Spark start reads its job config from an SSE-KMS bucket, and a start that is + refused leaves a stream's old run going.** A start reads `job.json` and the runner back from + S3 before it runs, as the scheduler's identity for a scheduled build, and the scheduler's + policy granted `s3:GetObject` on them and no `kms:Decrypt`: with a customer-managed key as the + artifact bucket's default encryption, every scheduled start failed with AccessDenied. The + policy now grants `kms:Decrypt` through S3 only (`kms:ViaService`), for those two objects, or + their bucket under an S3 Bucket Key. `fluid apply` also cancelled a stream's run of another + config before that read, so a start the deployed artifacts refused left the stream stopped + until the next apply. It now reads them first: a refusal cancels nothing, the old run keeps + going, and `fluid apply` exits 1. + ### Changed - **A Spark run starts only on the job config `fluid apply` compiled.** Every runtime, local diff --git a/fluid_build/build_runners/spark/_adapters.py b/fluid_build/build_runners/spark/_adapters.py index 2b5d69e2..afc8c664 100644 --- a/fluid_build/build_runners/spark/_adapters.py +++ b/fluid_build/build_runners/spark/_adapters.py @@ -265,10 +265,15 @@ def __init__(self, run: DesiredRun, clients: Optional[ClientFactory]) -> None: self.clients = clients # A local run has no AWS client: its "client" is the machine (``Local.host``). self.client = clients(self.service, run.region) if clients is not None else None + self._checked: Optional[DesiredRun] = None def checked(self) -> DesiredRun: - """:func:`checked_run` of this adapter's run.""" - return checked_run(self.run, self.clients) + """:func:`checked_run` of this adapter's run, read once: a reconcile checks before it + cancels the run of another config (a refusal leaves that run going), and the start + then uses the run it checked.""" + if self._checked is None: + self._checked = checked_run(self.run, self.clients) + return self._checked def runs(self, since: Optional[float] = None) -> List[RunInfo]: """The build's runs, newest first: every active one and the most recent ended ones, @@ -872,6 +877,10 @@ def runs(self, since: Optional[float] = None) -> List[RunInfo]: def get(self, run_id: str) -> RunInfo: return self._runs.get(run_id) + def checked(self) -> DesiredRun: + """Nothing is read back: each start writes this run's artifacts itself.""" + return self.run + def start(self, token: str, restart_of: Optional[str] = None) -> str: from fluid_build.build_runners.spark._local import start_run from fluid_build.spark import runner_source diff --git a/fluid_build/build_runners/spark/_reconcile.py b/fluid_build/build_runners/spark/_reconcile.py index c9e48a03..9e18be6b 100644 --- a/fluid_build/build_runners/spark/_reconcile.py +++ b/fluid_build/build_runners/spark/_reconcile.py @@ -13,7 +13,8 @@ one on the SAME checkpoint, so it resumes where the old one stopped. Spark's checkpoint holds the offsets of one topic on one cluster, so a new source on the old checkpoint is refused before anything is cancelled; moving ``properties.stream.checkpoint`` to a new - prefix is how a stream changes source. + prefix is how a stream changes source. A start the deployed job config or runner would + refuse (``_adapters.checked_run``) is refused before anything is cancelled too. 4. No running job: start one. When the newest run of this config ended, how it ended decides (:func:`_after_end`): @@ -480,6 +481,8 @@ def _reconcile_one( if only and confirm is not None and not confirm(_question(run, out, stale, matching, start)): out.action, out.message = "declined", _declined_text(stale, start) return out + if stale and not matching and not _checked_first(adapter, stale, out): + return out if not _cancel_and_wait(adapter, stale, out, clock, budget): return out if matching: # the new config already runs; the stale runs were extra @@ -552,6 +555,21 @@ def _question( return f"Spark {run.build} ({run.runtime}) on {out.where}: {what}?" +def _checked_first(adapter: Adapter, stale: Sequence[RunInfo], out: Outcome) -> bool: + """Read back the job config and runner the new run will start on + (:meth:`Adapter.checked`) before ``stale``, the runs of another config, are cancelled; + False (``out`` refused) when another apply has changed them: the old runs keep going.""" + try: + adapter.checked() + except SparkRunError as exc: + ids = ", ".join(s.run_id for s in stale) + they = f"run {ids} was" if len(stale) == 1 else f"runs {ids} were" + out.action, out.error = "refused", True + out.message = f"{_clean(exc)}; {they} left running" + return False + return True + + def _cancel_and_wait( adapter: Adapter, stale: Sequence[RunInfo], out: Outcome, clock: _Clock, budget: float ) -> bool: diff --git a/fluid_build/iac/providers/aws_identity.py b/fluid_build/iac/providers/aws_identity.py index 6a17081f..71d2b51c 100644 --- a/fluid_build/iac/providers/aws_identity.py +++ b/fluid_build/iac/providers/aws_identity.py @@ -1032,6 +1032,9 @@ class CallerRun: objects: Tuple[Any, ...] #: An isolated EMR Serverless application is found by name: ListApplications. lookup_application: bool = False + #: The ARN of the bucket ``objects`` are in: under an S3 Bucket Key, the encryption + #: context of their keys. + bucket: Any = None def _caller_targets(run: CallerRun, *, account: Any, region: Any) -> List[Any]: @@ -1062,7 +1065,11 @@ def caller_document(runs: Sequence[CallerRun], *, account: Any, region: Any) -> One statement per runtime (its actions once, on every target of the product's builds on that runtime), one PassRole per runtime, and one for the artifacts: a managed policy holds at most 6,144 characters, and a statement per build reached it at about six - builds. Each build's role and target is still named, so nothing else is granted.""" + builds. Each build's role and target is still named, so nothing else is granted. + + A start reads the job config and the runner back (``checked_run`` in + build_runners/spark/_adapters.py), so the artifacts get ``s3:GetObject`` and, through S3 + only, ``kms:Decrypt``: an SSE-KMS artifact bucket would refuse the read without it.""" statements: List[Dict[str, Any]] = [] runtimes = _unique([run.runtime for run in runs]) for runtime in runtimes: @@ -1126,6 +1133,37 @@ def caller_document(runs: Sequence[CallerRun], *, account: Any, region: Any) -> "resources": _unique([obj for run in runs for obj in run.objects]), } ) + # With an S3 Bucket Key the encryption context is the bucket's ARN; without one it is + # the object's. + contexts = _unique( + [ + context + for run in runs + for context in [*([run.bucket] if run.bucket is not None else []), *run.objects] + ] + ) + if contexts: + statements.append( + { + # As the job role's own S3Kms: whatever key the bucket encrypts them with. + "sid": "S3Kms", + "effect": "Allow", + "actions": ["kms:Decrypt"], + "resources": ["*"], + "condition": [ + { + "test": "StringEquals", + "variable": "kms:ViaService", + "values": [_join("s3.", region, ".amazonaws.com")], + }, + { + "test": "StringLike", + "variable": "kms:EncryptionContext:aws:s3:arn", + "values": contexts, + }, + ], + } + ) return {"statement": statements} diff --git a/fluid_build/iac/providers/aws_ingest.py b/fluid_build/iac/providers/aws_ingest.py index 1bf0f9bb..0835c56e 100644 --- a/fluid_build/iac/providers/aws_ingest.py +++ b/fluid_build/iac/providers/aws_ingest.py @@ -527,6 +527,7 @@ def _emit_build( aws_identity._join(bucket_arn, "/", b.runner_key), ), lookup_application=b.runtime == "emr-serverless" and not b.cfg.get("applicationId"), + bucket=bucket_arn, ) diff --git a/tests/build_runners/spark/test_adapters.py b/tests/build_runners/spark/test_adapters.py index 586f86e7..d19a257d 100644 --- a/tests/build_runners/spark/test_adapters.py +++ b/tests/build_runners/spark/test_adapters.py @@ -800,6 +800,13 @@ def test_a_config_this_apply_did_not_compile_starts_nothing(serverless, body): stub.assert_no_pending_responses() # nothing was sent to EMR +def test_the_check_before_a_cancel_and_the_start_read_the_artifacts_once(serverless): + adapter, _ = adapter_on(serverless, "emr-serverless") + bucket = adapter.clients("s3", serverless.region) + assert adapter.checked() is adapter.checked() + assert bucket.read == [serverless.job_uri, serverless.runner_uri] + + def test_a_runner_this_forge_does_not_ship_starts_nothing(serverless): adapter, stub = adapter_on(serverless, "emr-serverless", runner=b"import os # not ours\n") with pytest.raises(ad.SparkRunError, match="or its runner is not the one"): diff --git a/tests/build_runners/spark/test_reconcile.py b/tests/build_runners/spark/test_reconcile.py index e1c6401d..878590f9 100644 --- a/tests/build_runners/spark/test_reconcile.py +++ b/tests/build_runners/spark/test_reconcile.py @@ -6,6 +6,8 @@ from __future__ import annotations import dataclasses +import io +import logging import pytest @@ -13,7 +15,8 @@ ClientError = pytest.importorskip("botocore.exceptions").ClientError from fluid_build.build_runners.spark import _reconcile as rc -from fluid_build.build_runners.spark._adapters import RunInfo, SparkRunError +from fluid_build.build_runners.spark._adapters import RunInfo, SparkRunError, checked_run +from fluid_build.cli import _spark_apply ENV = {rc.WAIT_ENV: "5"} @@ -70,6 +73,9 @@ def __init__( def where(self): return "fake cluster" + def checked(self): + return self.run + def runs(self, since=None): self.since.append(since) if self.later is not None and "start" in self.calls: @@ -143,6 +149,37 @@ def test_another_config_is_cancelled_waited_for_and_replaced(run, clock, use): assert fake.calls == ["cancel old1", "start"] +class DeployedConfig: + """The artifact bucket after another apply: its job.json is not the one this apply + compiled.""" + + def get_object(self, Bucket, Key): # noqa: N803 - boto3's names + return {"Body": io.BytesIO(b'{"mode": "stream", "another": "config"}')} + + +def refusing(fake): + """``fake`` with the adapters' real check of the deployed artifacts.""" + fake.checked = lambda: checked_run(fake.run, lambda service, region: DeployedConfig()) + return fake + + +def test_a_start_the_deployed_config_refuses_leaves_the_old_run_going(run, clock, use): + fake = refusing(FakeAdapter([info("old1", "RUNNING", hash_="stale")])) + use(fake) + out = go(run, clock) + assert (out.action, out.error, out.cancelled, fake.calls) == ("refused", True, [], []) + assert "is not the one this apply compiled" in out.message + assert out.message.endswith("; run old1 was left running") + + +def test_fluid_apply_fails_when_the_deployed_config_refuses_the_start(run, use): + fake = refusing(FakeAdapter([info("old1", "RUNNING", hash_="stale")])) + use(fake) + log = logging.getLogger("test.spark.refused") + assert _spark_apply.reconcile_after_apply([run], log, dry_run=False) == 1 + assert fake.calls == [] # nothing cancelled, nothing started + + def test_a_new_topic_on_the_same_checkpoint_is_refused_before_anything_is_cancelled( run, clock, use ): diff --git a/tests/iac/test_iac_aws_spark_infra.py b/tests/iac/test_iac_aws_spark_infra.py index ff8e0e05..d2a9b230 100644 --- a/tests/iac/test_iac_aws_spark_infra.py +++ b/tests/iac/test_iac_aws_spark_infra.py @@ -829,6 +829,58 @@ def test_a_run_to_completion_build_gets_the_schedulers_policy(contract): assert not any(s.startswith("FindApplication") for s in sids) +def test_the_scheduler_may_decrypt_the_artifacts_it_reads_back_through_s3_only(contract): + """A start reads job.json and the runner back (``_adapters.checked_run``); an SSE-KMS + artifact bucket refuses that read without ``kms:Decrypt``.""" + contract["builds"][0]["properties"]["stream"]["trigger"] = {"availableNow": True} + _, data = emit(contract) + document = data["aws_iam_policy_document"]["commerce_orders_stream_spark_caller"] + kms = statement(document, "S3Kms") + assert kms["actions"] == ["kms:Decrypt"] + assert kms["resources"] == ["*"] + via, context = kms["condition"] + assert (via["test"], via["variable"]) == ("StringEquals", "kms:ViaService") + assert via["values"] == [ + f"s3.${{data.aws_region.{aws_ingest.REGION_DATA}.region}}.amazonaws.com" + ] + assert (context["test"], context["variable"]) == ( + "StringLike", + "kms:EncryptionContext:aws:s3:arn", + ) + artifacts = next(s for s in document["statement"] if s["sid"].startswith("Artifacts")) + # Under an S3 Bucket Key the context is the bucket's ARN; without one, the object's. + assert context["values"] == ["arn:aws:s3:::commerce-lake", *artifacts["resources"]] + + +def _iam_policy_chars(document): + """The policy as IAM counts it: its JSON without whitespace (OpenTofu expressions are + counted as written, longer than the account ids and regions they resolve to).""" + statements = [] + for s in document["statement"]: + rendered = { + "Sid": s["sid"], + "Effect": s["effect"], + "Action": s["actions"], + "Resource": s["resources"], + } + for c in s.get("condition") or []: + rendered.setdefault("Condition", {}).setdefault(c["test"], {})[c["variable"]] = c[ + "values" + ] + statements.append(rendered) + policy = {"Version": "2012-10-17", "Statement": statements} + return len(json.dumps(policy, separators=(",", ":"))) + + +@pytest.mark.parametrize("runtime", ["emr-serverless", "glue", "emr-eks", "emr-ec2"]) +def test_the_schedulers_policy_fits_a_managed_policy_on_every_runtime(contract, runtime): + contract["builds"][0]["properties"]["stream"]["trigger"] = {"availableNow": True} + _, data = emit(_runtime(contract, runtime)) + document = data["aws_iam_policy_document"]["commerce_orders_stream_spark_caller"] + assert statement(document, "S3Kms") + assert _iam_policy_chars(document) < 6144 # IAM's limit for a managed policy + + # --- Lake Formation writes (item 4) --------------------------------------------------- @@ -1196,12 +1248,14 @@ def test_the_schedulers_policy_names_each_action_once_per_runtime(): "RunEmrEc2", "StepRoleEmrEc2", "Artifacts", + "S3Kms", ] by_sid = {s["sid"]: s for s in doc["statement"]} serverless = by_sid["RunEmrServerless"]["resources"] assert len(serverless) == 4 # two applications, each named once with its job runs assert len(by_sid["PassRoleEmrServerless"]["resources"]) == 8 assert len(by_sid["Artifacts"]["resources"]) == 9 + assert by_sid["S3Kms"]["condition"][1]["values"] == by_sid["Artifacts"]["resources"] assert len(json.dumps(doc, separators=(",", ":"))) < 6144