Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions .github/workflows/iac-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -205,13 +205,15 @@ jobs:
# A Glue table's partition keys and Athena projection: whether a changed
# layout replaces a live table, and dropping the keys updates it in place,
# depends on how `replace_triggered_by` meets the trigger's create, update
# and destroy, so only a real plan against an applied table proves it.
# and destroy, so only a real plan against an applied table proves it. So
# does an Iceberg table's plan after an engine's commit rewrote its location.
- name: "Glue partition keys apply + plan (tofu vs moto, creds-free)"
tests: tests/iac/test_iac_aws_partition_plan_moto.py
tests: tests/iac/test_iac_aws_partition_plan_moto.py tests/iac/test_iac_aws_iceberg_location_moto.py
pytest-args: ""
junit: glue-partition-moto.xml
require: |
Glue partition moto=tests/iac/test_iac_aws_partition_plan_moto.py
Iceberg location moto=tests/iac/test_iac_aws_iceberg_location_moto.py
# `fluid diff` refreshes the apply's OpenTofu state: a real apply,
# then the refresh plan, against moto. Whether a clean apply reads as
# no drift depends on what the provider reads back, so only a real
Expand Down
2 changes: 1 addition & 1 deletion .secrets.baseline

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 8 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
The artifact bucket's write access stays the boundary. A scheduler's role already reads both
objects. The runner changed, so each running Spark stream restarts once on its next apply.

- **An Iceberg table on Glue plans no change after an engine commits to it.** forge emitted the
table's location with a trailing slash; Iceberg keeps it without one and writes it back into
Glue on every commit. So each `fluid apply` after a commit (every scheduled run beside a
stream) planned `~1` per written table and sent an `UpdateTable` while the stream kept
committing, with `previous_metadata_location` from the plan's refresh. The location is now
emitted as Iceberg keeps it. The first apply after the upgrade updates each existing Iceberg
table's location once, in place; Hive-format tables keep their trailing slash.

## [0.33.0] — 2026-11-19

### Added
Expand Down
12 changes: 10 additions & 2 deletions fluid_build/iac/providers/aws.py
Original file line number Diff line number Diff line change
Expand Up @@ -1316,10 +1316,18 @@ def _emit_glue(
# the renderer escapes at render time. Fallback is decided on the raw input
# (``bucket_uses_fallback``), which a contract cannot spoof.
bucket, path = _warehouse.normalize_location(loc, account_ref=_CALLER_ACCOUNT_TOKEN)
if is_iceberg_format(fmt):
# Iceberg keeps a table's location without the trailing slash
# (``LocationUtil.stripTrailingSlash``), and every commit writes it back into Glue
# (``GlueTableOperations``). With the slash here, each plan after a commit updated
# the table to put it back: an ``UpdateTable`` beside a running stream's commits.
path = path.rstrip("/")
# ``s3://bucket`` for an Iceberg table at a bucket's root, as Iceberg keeps that too.
sep = "/" if path or not is_iceberg_format(fmt) else ""
if _warehouse.bucket_uses_fallback(loc):
storage["location"] = TofuExpr(f"s3://{bucket}/{_literal(path)}")
storage["location"] = TofuExpr(f"s3://{bucket}{sep}{_literal(path)}")
else:
storage["location"] = f"s3://{bucket}/{path}"
storage["location"] = f"s3://{bucket}{sep}{path}"
parameters: Dict[str, str] = {"classification": fmt, "managed_by": "fluid"}
if "iceberg" in str(fmt).lower():
# AWS Glue / Athena identify an Iceberg table via this parameter. For a Glue-
Expand Down
4 changes: 3 additions & 1 deletion fluid_build/providers/aws/plan/planner.py
Original file line number Diff line number Diff line change
Expand Up @@ -631,7 +631,9 @@ def _plan_exposures(
"database": database,
"table": table,
"columns": glue_columns,
"location": s3_location,
"location": _warehouse.iceberg_table_location(
location, account_ref=account_id
),
"icebergConfig": iceberg_config,
"description": exposure.get("description"),
}
Expand Down
7 changes: 7 additions & 0 deletions fluid_build/providers/aws/util/warehouse.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,13 @@ def get_iceberg_warehouse(loc: Mapping[str, Any], *, account_ref: str) -> str:
return f"s3://{bucket}/{path}"


def iceberg_table_location(loc: Mapping[str, Any], *, account_ref: str) -> str:
"""An Iceberg table's location as Iceberg keeps it: :func:`get_iceberg_warehouse`
without its trailing slash (``LocationUtil.stripTrailingSlash``), which is what every
engine's commit writes back into the Glue table (``GlueTableOperations``)."""
return get_iceberg_warehouse(loc, account_ref=account_ref).rstrip("/")


def bucket_uses_fallback(loc: Mapping[str, Any]) -> bool:
"""True when the bucket is absent / an unresolved ``{{ env.* }}`` template,
so the warehouse falls back to the ``{account_ref}-fluid-data`` bucket.
Expand Down
3 changes: 2 additions & 1 deletion tests/iac/test_iac_aws_iceberg_catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,8 @@ def test_iceberg_table_format_spelling_is_the_same_table(self):
"parameters": {"classification": "iceberg", "managed_by": "fluid"},
"storage_descriptor": {
"columns": [{"name": "order_id", "type": "string"}, {"name": "qty", "type": "int"}],
"location": "s3://lake/orders/",
# Iceberg keeps its location without the trailing slash.
"location": "s3://lake/orders",
},
"open_table_format_input": {
"iceberg_input": {"metadata_operation": "CREATE", "version": "2"}
Expand Down
76 changes: 76 additions & 0 deletions tests/iac/test_iac_aws_iceberg_location_moto.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
# Copyright 2025 Agentics Transformation Limited
# SPDX-License-Identifier: Apache-2.0

"""An Iceberg table's Glue location after an engine's commit plans no change, with the real
``tofu`` against moto.

Iceberg keeps a table's location without the trailing slash, and every commit writes it back
into Glue (``GlueTableOperations``) beside the new ``metadata_location``. forge emitted the
location with the slash, so each plan after a commit updated the table to put the slash back:
an ``UpdateTable`` beside a running stream's commits. The commit is played here as Iceberg
makes it, through boto3. Skipped unless ``tofu`` is on PATH and moto's ``server`` extra is
installed; ``iac-tests.yml`` Stage 1 installs both and fails if this file only skipped.
"""

from __future__ import annotations

import re
from pathlib import Path

import pytest

from fluid_build.cli import _apply_opentofu_engine as engine
from tests.iac.test_iac_aws_apply_engine_moto import ( # noqa: F401 - moto_endpoint is a fixture
_LOG,
_SKIP,
_SKIP_REASON,
_apply_args,
_aws_iceberg_contract,
_boto,
_drop_moto_provider_sidecar,
_write_plan,
moto_endpoint,
)

pytestmark = [pytest.mark.integration, pytest.mark.provider, pytest.mark.aws]


def _commit_as_iceberg_does(glue, location: str) -> None:
"""What ``GlueTableOperations.doCommit`` sends: the table as it is, its location as
Iceberg keeps it, and the new metadata file."""
table = glue.get_table(DatabaseName="engine_silver", Name="events")["Table"]
storage = dict(table["StorageDescriptor"], Location=location)
parameters = dict(
table.get("Parameters") or {},
metadata_location=f"{location}/metadata/00001-commit.metadata.json",
previous_metadata_location=f"{location}/metadata/00000-create.metadata.json",
)
glue.update_table(
DatabaseName="engine_silver",
TableInput={
"Name": "events",
"TableType": table.get("TableType", "EXTERNAL_TABLE"),
"StorageDescriptor": storage,
"Parameters": parameters,
},
)


@pytest.mark.skipif(_SKIP, reason=_SKIP_REASON)
def test_a_plan_after_an_iceberg_commit_changes_nothing(
moto_endpoint: str, tmp_path: Path, capsys
) -> None:
cid = "iac.engine.iceberg-location"
plan = _write_plan(tmp_path / "plan.json", _aws_iceberg_contract("iceberg-location-lake", cid))
_drop_moto_provider_sidecar(tmp_path, cid, moto_endpoint)
assert engine.apply_via_opentofu(_apply_args(plan, tmp_path), _LOG) == 0

glue = _boto("glue", moto_endpoint)
emitted = glue.get_table(DatabaseName="engine_silver", Name="events")["Table"]
assert emitted["StorageDescriptor"]["Location"] == "s3://iceberg-location-lake/silver/events"

_commit_as_iceberg_does(glue, "s3://iceberg-location-lake/silver/events")
capsys.readouterr()
assert engine.apply_via_opentofu(_apply_args(plan, tmp_path, dry_run=True), _LOG) == 0
summary = re.search(r"tofu plan: \+(\d+) ~(\d+) -(\d+)", capsys.readouterr().out)
assert summary is not None and summary.groups() == ("0", "0", "0")
20 changes: 16 additions & 4 deletions tests/iac/test_iac_aws_iceberg_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,8 +140,9 @@ def test_a_parquet_table_is_unchanged(contract):

@pytest.mark.integration
@pytest.mark.skipif(_TOFU is None, reason="tofu not on PATH")
def test_a_table_created_before_metadata_plans_no_change(tmp_path):
"""No ForceNew replace, so the data-loss gate has nothing to stop."""
def test_a_table_created_before_metadata_is_updated_in_place_not_replaced(tmp_path):
"""No ForceNew replace, so the data-loss gate has nothing to stop. The one change is the
location, which loses its trailing slash as Iceberg keeps it, in place."""
from tests.iac.test_iac_tofu_validate import _tofu_init_or_skip

plugin = get_iac_plugin("aws")
Expand Down Expand Up @@ -179,6 +180,17 @@ def test_a_table_created_before_metadata_plans_no_change(tmp_path):
table_address = (
"aws_glue_catalog_table.lane_sparkinfra_ice_upgrade_sparkinfra_ice_upgrade_events"
)
assert actions[table_address] == ["no-op"], actions
assert all(a == ["no-op"] for a in actions.values()), actions
assert actions[table_address] == ["update"], actions
change = next(rc["change"] for rc in plan["resource_changes"] if rc["address"] == table_address)
before = change["before"]["storage_descriptor"][0]
after = change["after"]["storage_descriptor"][0]
assert (before["location"], after["location"]) == (
after["location"] + "/",
before["location"].rstrip("/"),
)
assert {k: v for k, v in before.items() if k != "location"} == {
k: v for k, v in after.items() if k != "location"
}
others = {address: a for address, a in actions.items() if address != table_address}
assert all(a == ["no-op"] for a in others.values()), actions
assert plugin.plan_data_changes(plan) == []
10 changes: 8 additions & 2 deletions tests/iac/test_iac_aws_spark_dead_letters.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,13 @@ def test_the_dead_letters_table_sits_beside_the_output_never_inside_it(contract)
table = resources["aws_glue_catalog_table"][TABLE_KEY]
assert table["name"] == "order_lines__dead_letter"
storage = table["storage_descriptor"]
assert storage["location"] == "s3://commerce-lake/iceberg/commerce/order_lines__dead_letter/"
# As Iceberg keeps a table's location: no trailing slash (the S3 grant below has its own).
assert storage["location"] == "s3://commerce-lake/iceberg/commerce/order_lines__dead_letter"
# Beside, never inside, by whole path segments (``order_lines__dead_letter`` starts with
# ``order_lines`` as a string, not as a directory).
output = resources["aws_glue_catalog_table"]["commerce_orders_stream_commerce_order_lines"]
outside = output["storage_descriptor"]["location"].rstrip("/") + "/"
assert not (storage["location"].rstrip("/") + "/").startswith(outside)
assert {c["name"]: c["type"] for c in storage["columns"]} == {
"error": "string",
"raw_value": "binary",
Expand Down Expand Up @@ -107,7 +113,7 @@ def test_the_dead_letters_table_sits_beside_the_output_never_inside_it(contract)
def test_a_named_dead_letters_table_is_the_one_emitted_and_granted(contract):
resources, data = emit(with_dead_letters(contract, table="order_rejects"))
table = resources["aws_glue_catalog_table"]["commerce_orders_stream_commerce_order_rejects"]
assert table["storage_descriptor"]["location"].endswith("/iceberg/commerce/order_rejects/")
assert table["storage_descriptor"]["location"].endswith("/iceberg/commerce/order_rejects")
glue = statement(data["aws_iam_policy_document"][f"{KEY}_access"], "GlueTable")
assert glue["resources"][-1].endswith(":table/commerce/order_rejects")

Expand Down
8 changes: 7 additions & 1 deletion tests/providers/aws/test_warehouse_derivation.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,11 +147,17 @@ def test_explicit_bucket_byte_stable_no_extra_data_source():
plugin = get_iac_plugin("aws")
tbl = next(iter(plugin.emit(contract)["aws_glue_catalog_table"].values()))
loc = tbl["storage_descriptor"]["location"]
assert loc == "s3://lake/sales/orders/"
assert loc == "s3://lake/sales/orders" # as Iceberg keeps it: no trailing slash
assert not isinstance(loc, TofuExpr)
assert "aws_caller_identity" not in plugin.emit_data(contract)


def test_an_iceberg_table_at_a_bucket_root_has_no_trailing_slash_either():
location = {"database": "sales", "table": "orders", "bucket": "lake", "path": ""}
assert _iac_location(location) == "s3://lake"
assert _planner_location(location) == "s3://lake"


def test_same_warehouse_logical_equality():
from fluid_build.providers.aws.util.warehouse import same_warehouse

Expand Down
Loading