Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
`emr-serverless:StopApplication`, which the identity running the destroy needs beyond the
runtime's list and cancel actions.

- **A scheduled Spark start reads its job config from an SSE-KMS bucket, and a start that is
refused leaves a stream's old run going.** A start reads `job.json` and the runner back from
S3 before it runs, as the scheduler's identity for a scheduled build, and the scheduler's
policy granted `s3:GetObject` on them and no `kms:Decrypt`: with a customer-managed key as the
artifact bucket's default encryption, every scheduled start failed with AccessDenied. The
policy now grants `kms:Decrypt` through S3 only (`kms:ViaService`), for those two objects, or
their bucket under an S3 Bucket Key. `fluid apply` also cancelled a stream's run of another
config before that read, so a start the deployed artifacts refused left the stream stopped
until the next apply. It now reads them first: a refusal cancels nothing, the old run keeps
going, and `fluid apply` exits 1.

### Changed

- **A Spark run starts only on the job config `fluid apply` compiled.** Every runtime, local
Expand Down
13 changes: 11 additions & 2 deletions fluid_build/build_runners/spark/_adapters.py
Original file line number Diff line number Diff line change
Expand Up @@ -265,10 +265,15 @@ def __init__(self, run: DesiredRun, clients: Optional[ClientFactory]) -> None:
self.clients = clients
# A local run has no AWS client: its "client" is the machine (``Local.host``).
self.client = clients(self.service, run.region) if clients is not None else None
self._checked: Optional[DesiredRun] = None

def checked(self) -> DesiredRun:
""":func:`checked_run` of this adapter's run."""
return checked_run(self.run, self.clients)
""":func:`checked_run` of this adapter's run, read once: a reconcile checks before it
cancels the run of another config (a refusal leaves that run going), and the start
then uses the run it checked."""
if self._checked is None:
self._checked = checked_run(self.run, self.clients)
return self._checked

def runs(self, since: Optional[float] = None) -> List[RunInfo]:
"""The build's runs, newest first: every active one and the most recent ended ones,
Expand Down Expand Up @@ -872,6 +877,10 @@ def runs(self, since: Optional[float] = None) -> List[RunInfo]:
def get(self, run_id: str) -> RunInfo:
return self._runs.get(run_id)

def checked(self) -> DesiredRun:
"""Nothing is read back: each start writes this run's artifacts itself."""
return self.run

def start(self, token: str, restart_of: Optional[str] = None) -> str:
from fluid_build.build_runners.spark._local import start_run
from fluid_build.spark import runner_source
Expand Down
20 changes: 19 additions & 1 deletion fluid_build/build_runners/spark/_reconcile.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@
one on the SAME checkpoint, so it resumes where the old one stopped. Spark's checkpoint
holds the offsets of one topic on one cluster, so a new source on the old checkpoint is
refused before anything is cancelled; moving ``properties.stream.checkpoint`` to a new
prefix is how a stream changes source.
prefix is how a stream changes source. A start the deployed job config or runner would
refuse (``_adapters.checked_run``) is refused before anything is cancelled too.
4. No running job: start one. When the newest run of this config ended, how it ended
decides (:func:`_after_end`):

Expand Down Expand Up @@ -480,6 +481,8 @@ def _reconcile_one(
if only and confirm is not None and not confirm(_question(run, out, stale, matching, start)):
out.action, out.message = "declined", _declined_text(stale, start)
return out
if stale and not matching and not _checked_first(adapter, stale, out):
return out
if not _cancel_and_wait(adapter, stale, out, clock, budget):
return out
if matching: # the new config already runs; the stale runs were extra
Expand Down Expand Up @@ -552,6 +555,21 @@ def _question(
return f"Spark {run.build} ({run.runtime}) on {out.where}: {what}?"


def _checked_first(adapter: Adapter, stale: Sequence[RunInfo], out: Outcome) -> bool:
"""Read back the job config and runner the new run will start on
(:meth:`Adapter.checked`) before ``stale``, the runs of another config, are cancelled;
False (``out`` refused) when another apply has changed them: the old runs keep going."""
try:
adapter.checked()
except SparkRunError as exc:
ids = ", ".join(s.run_id for s in stale)
they = f"run {ids} was" if len(stale) == 1 else f"runs {ids} were"
out.action, out.error = "refused", True
out.message = f"{_clean(exc)}; {they} left running"
return False
return True


def _cancel_and_wait(
adapter: Adapter, stale: Sequence[RunInfo], out: Outcome, clock: _Clock, budget: float
) -> bool:
Expand Down
40 changes: 39 additions & 1 deletion fluid_build/iac/providers/aws_identity.py
Original file line number Diff line number Diff line change
Expand Up @@ -1032,6 +1032,9 @@ class CallerRun:
objects: Tuple[Any, ...]
#: An isolated EMR Serverless application is found by name: ListApplications.
lookup_application: bool = False
#: The ARN of the bucket ``objects`` are in: under an S3 Bucket Key, the encryption
#: context of their keys.
bucket: Any = None


def _caller_targets(run: CallerRun, *, account: Any, region: Any) -> List[Any]:
Expand Down Expand Up @@ -1062,7 +1065,11 @@ def caller_document(runs: Sequence[CallerRun], *, account: Any, region: Any) ->
One statement per runtime (its actions once, on every target of the product's builds on
that runtime), one PassRole per runtime, and one for the artifacts: a managed policy
holds at most 6,144 characters, and a statement per build reached it at about six
builds. Each build's role and target is still named, so nothing else is granted."""
builds. Each build's role and target is still named, so nothing else is granted.

A start reads the job config and the runner back (``checked_run`` in
build_runners/spark/_adapters.py), so the artifacts get ``s3:GetObject`` and, through S3
only, ``kms:Decrypt``: an SSE-KMS artifact bucket would refuse the read without it."""
statements: List[Dict[str, Any]] = []
runtimes = _unique([run.runtime for run in runs])
for runtime in runtimes:
Expand Down Expand Up @@ -1126,6 +1133,37 @@ def caller_document(runs: Sequence[CallerRun], *, account: Any, region: Any) ->
"resources": _unique([obj for run in runs for obj in run.objects]),
}
)
# With an S3 Bucket Key the encryption context is the bucket's ARN; without one it is
# the object's.
contexts = _unique(
[
context
for run in runs
for context in [*([run.bucket] if run.bucket is not None else []), *run.objects]
]
)
if contexts:
statements.append(
{
# As the job role's own S3Kms: whatever key the bucket encrypts them with.
"sid": "S3Kms",
"effect": "Allow",
"actions": ["kms:Decrypt"],
"resources": ["*"],
"condition": [
{
"test": "StringEquals",
"variable": "kms:ViaService",
"values": [_join("s3.", region, ".amazonaws.com")],
},
{
"test": "StringLike",
"variable": "kms:EncryptionContext:aws:s3:arn",
"values": contexts,
},
],
}
)
return {"statement": statements}


Expand Down
1 change: 1 addition & 0 deletions fluid_build/iac/providers/aws_ingest.py
Original file line number Diff line number Diff line change
Expand Up @@ -527,6 +527,7 @@ def _emit_build(
aws_identity._join(bucket_arn, "/", b.runner_key),
),
lookup_application=b.runtime == "emr-serverless" and not b.cfg.get("applicationId"),
bucket=bucket_arn,
)


Expand Down
7 changes: 7 additions & 0 deletions tests/build_runners/spark/test_adapters.py
Original file line number Diff line number Diff line change
Expand Up @@ -800,6 +800,13 @@ def test_a_config_this_apply_did_not_compile_starts_nothing(serverless, body):
stub.assert_no_pending_responses() # nothing was sent to EMR


def test_the_check_before_a_cancel_and_the_start_read_the_artifacts_once(serverless):
adapter, _ = adapter_on(serverless, "emr-serverless")
bucket = adapter.clients("s3", serverless.region)
assert adapter.checked() is adapter.checked()
assert bucket.read == [serverless.job_uri, serverless.runner_uri]


def test_a_runner_this_forge_does_not_ship_starts_nothing(serverless):
adapter, stub = adapter_on(serverless, "emr-serverless", runner=b"import os # not ours\n")
with pytest.raises(ad.SparkRunError, match="or its runner is not the one"):
Expand Down
39 changes: 38 additions & 1 deletion tests/build_runners/spark/test_reconcile.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,17 @@
from __future__ import annotations

import dataclasses
import io
import logging

import pytest

# botocore comes with boto3, the [aws] extra; the reconcile uses its ClientError shape.
ClientError = pytest.importorskip("botocore.exceptions").ClientError

from fluid_build.build_runners.spark import _reconcile as rc
from fluid_build.build_runners.spark._adapters import RunInfo, SparkRunError
from fluid_build.build_runners.spark._adapters import RunInfo, SparkRunError, checked_run
from fluid_build.cli import _spark_apply

ENV = {rc.WAIT_ENV: "5"}

Expand Down Expand Up @@ -70,6 +73,9 @@ def __init__(
def where(self):
return "fake cluster"

def checked(self):
return self.run

def runs(self, since=None):
self.since.append(since)
if self.later is not None and "start" in self.calls:
Expand Down Expand Up @@ -143,6 +149,37 @@ def test_another_config_is_cancelled_waited_for_and_replaced(run, clock, use):
assert fake.calls == ["cancel old1", "start"]


class DeployedConfig:
"""The artifact bucket after another apply: its job.json is not the one this apply
compiled."""

def get_object(self, Bucket, Key): # noqa: N803 - boto3's names
return {"Body": io.BytesIO(b'{"mode": "stream", "another": "config"}')}


def refusing(fake):
"""``fake`` with the adapters' real check of the deployed artifacts."""
fake.checked = lambda: checked_run(fake.run, lambda service, region: DeployedConfig())
return fake


def test_a_start_the_deployed_config_refuses_leaves_the_old_run_going(run, clock, use):
fake = refusing(FakeAdapter([info("old1", "RUNNING", hash_="stale")]))
use(fake)
out = go(run, clock)
assert (out.action, out.error, out.cancelled, fake.calls) == ("refused", True, [], [])
assert "is not the one this apply compiled" in out.message
assert out.message.endswith("; run old1 was left running")


def test_fluid_apply_fails_when_the_deployed_config_refuses_the_start(run, use):
fake = refusing(FakeAdapter([info("old1", "RUNNING", hash_="stale")]))
use(fake)
log = logging.getLogger("test.spark.refused")
assert _spark_apply.reconcile_after_apply([run], log, dry_run=False) == 1
assert fake.calls == [] # nothing cancelled, nothing started


def test_a_new_topic_on_the_same_checkpoint_is_refused_before_anything_is_cancelled(
run, clock, use
):
Expand Down
54 changes: 54 additions & 0 deletions tests/iac/test_iac_aws_spark_infra.py
Original file line number Diff line number Diff line change
Expand Up @@ -829,6 +829,58 @@ def test_a_run_to_completion_build_gets_the_schedulers_policy(contract):
assert not any(s.startswith("FindApplication") for s in sids)


def test_the_scheduler_may_decrypt_the_artifacts_it_reads_back_through_s3_only(contract):
"""A start reads job.json and the runner back (``_adapters.checked_run``); an SSE-KMS
artifact bucket refuses that read without ``kms:Decrypt``."""
contract["builds"][0]["properties"]["stream"]["trigger"] = {"availableNow": True}
_, data = emit(contract)
document = data["aws_iam_policy_document"]["commerce_orders_stream_spark_caller"]
kms = statement(document, "S3Kms")
assert kms["actions"] == ["kms:Decrypt"]
assert kms["resources"] == ["*"]
via, context = kms["condition"]
assert (via["test"], via["variable"]) == ("StringEquals", "kms:ViaService")
assert via["values"] == [
f"s3.${{data.aws_region.{aws_ingest.REGION_DATA}.region}}.amazonaws.com"
]
assert (context["test"], context["variable"]) == (
"StringLike",
"kms:EncryptionContext:aws:s3:arn",
)
artifacts = next(s for s in document["statement"] if s["sid"].startswith("Artifacts"))
# Under an S3 Bucket Key the context is the bucket's ARN; without one, the object's.
assert context["values"] == ["arn:aws:s3:::commerce-lake", *artifacts["resources"]]


def _iam_policy_chars(document):
"""The policy as IAM counts it: its JSON without whitespace (OpenTofu expressions are
counted as written, longer than the account ids and regions they resolve to)."""
statements = []
for s in document["statement"]:
rendered = {
"Sid": s["sid"],
"Effect": s["effect"],
"Action": s["actions"],
"Resource": s["resources"],
}
for c in s.get("condition") or []:
rendered.setdefault("Condition", {}).setdefault(c["test"], {})[c["variable"]] = c[
"values"
]
statements.append(rendered)
policy = {"Version": "2012-10-17", "Statement": statements}
return len(json.dumps(policy, separators=(",", ":")))


@pytest.mark.parametrize("runtime", ["emr-serverless", "glue", "emr-eks", "emr-ec2"])
def test_the_schedulers_policy_fits_a_managed_policy_on_every_runtime(contract, runtime):
contract["builds"][0]["properties"]["stream"]["trigger"] = {"availableNow": True}
_, data = emit(_runtime(contract, runtime))
document = data["aws_iam_policy_document"]["commerce_orders_stream_spark_caller"]
assert statement(document, "S3Kms")
assert _iam_policy_chars(document) < 6144 # IAM's limit for a managed policy


# --- Lake Formation writes (item 4) ---------------------------------------------------


Expand Down Expand Up @@ -1196,12 +1248,14 @@ def test_the_schedulers_policy_names_each_action_once_per_runtime():
"RunEmrEc2",
"StepRoleEmrEc2",
"Artifacts",
"S3Kms",
]
by_sid = {s["sid"]: s for s in doc["statement"]}
serverless = by_sid["RunEmrServerless"]["resources"]
assert len(serverless) == 4 # two applications, each named once with its job runs
assert len(by_sid["PassRoleEmrServerless"]["resources"]) == 8
assert len(by_sid["Artifacts"]["resources"]) == 9
assert by_sid["S3Kms"]["condition"][1]["values"] == by_sid["Artifacts"]["resources"]
assert len(json.dumps(doc, separators=(",", ":"))) < 6144


Expand Down
Loading