From a519b2d2e61e167f5f639d82e5ff0de67d2dcac7 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 19 Nov 2026 07:00:58 +0100 Subject: [PATCH 1/3] fix(iac): emit an Iceberg table's Glue location as Iceberg keeps it forge emitted an Iceberg table's location with a trailing slash; Iceberg keeps it without one (LocationUtil.stripTrailingSlash) and writes it back into Glue on every commit (GlueTableOperations). So each fluid apply after a commit planned ~1 per written table and sent an UpdateTable while a stream kept committing, with previous_metadata_location from the plan's refresh. The Glue table location of an Iceberg table, in the OpenTofu emit and in the native planner's glue.ensure_iceberg_table action alike, is now the warehouse location without its trailing slash (warehouse.iceberg_table_location). Hive-format tables keep the slash. A real tofu run against moto plays a commit as GlueTableOperations makes it, then plans: +0 ~0 -0 (the parent emit planned +0 ~1 -0). --- .github/workflows/iac-tests.yml | 6 +- .secrets.baseline | 4 +- CHANGELOG.md | 8 ++ fluid_build/iac/providers/aws.py | 6 ++ fluid_build/providers/aws/plan/planner.py | 4 +- fluid_build/providers/aws/util/warehouse.py | 7 ++ tests/iac/test_iac_aws_iceberg_catalog.py | 3 +- .../iac/test_iac_aws_iceberg_location_moto.py | 76 +++++++++++++++++++ .../aws/test_warehouse_derivation.py | 2 +- 9 files changed, 109 insertions(+), 7 deletions(-) create mode 100644 tests/iac/test_iac_aws_iceberg_location_moto.py diff --git a/.github/workflows/iac-tests.yml b/.github/workflows/iac-tests.yml index 49590f3c6..95b087f76 100644 --- a/.github/workflows/iac-tests.yml +++ b/.github/workflows/iac-tests.yml @@ -193,13 +193,15 @@ jobs: # A Glue table's partition keys and Athena projection: whether a changed # layout replaces a live table, and dropping the keys updates it in place, # depends on how `replace_triggered_by` meets the trigger's create, update - # and destroy, so only a real plan against an applied table proves it. + # and destroy, so only a real plan against an applied table proves it. So + # does an Iceberg table's plan after an engine's commit rewrote its location. - name: "Glue partition keys apply + plan (tofu vs moto, creds-free)" - tests: tests/iac/test_iac_aws_partition_plan_moto.py + tests: tests/iac/test_iac_aws_partition_plan_moto.py tests/iac/test_iac_aws_iceberg_location_moto.py pytest-args: "" junit: glue-partition-moto.xml require: | Glue partition moto=tests/iac/test_iac_aws_partition_plan_moto.py + Iceberg location moto=tests/iac/test_iac_aws_iceberg_location_moto.py # `fluid diff` refreshes the apply's OpenTofu state: a real apply, # then the refresh plan, against moto. Whether a clean apply reads as # no drift depends on what the provider reads back, so only a real diff --git a/.secrets.baseline b/.secrets.baseline index aa90b29c8..1b620f081 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -139,7 +139,7 @@ "filename": ".github/workflows/iac-tests.yml", "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", "is_verified": false, - "line_number": 429 + "line_number": 431 } ], ".github/workflows/integration.yml": [ @@ -1915,5 +1915,5 @@ } ] }, - "generated_at": "2026-11-19T05:23:51Z" + "generated_at": "2026-11-19T05:53:31Z" } diff --git a/CHANGELOG.md b/CHANGELOG.md index bb69af4fb..a5dea515e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -459,6 +459,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **An Iceberg table on Glue plans no change after an engine commits to it.** forge emitted the + table's location with a trailing slash; Iceberg keeps it without one and writes it back into + Glue on every commit. So each `fluid apply` after a commit (every scheduled run beside a + stream) planned `~1` per written table and sent an `UpdateTable` while the stream kept + committing, with `previous_metadata_location` from the plan's refresh. The location is now + emitted as Iceberg keeps it. The first apply after the upgrade updates each existing Iceberg + table's location once, in place; Hive-format tables keep their trailing slash. + - **An EMR on EC2 stream starts only once its cluster's role carries the build's policy.** A step runs as the cluster's instance-profile role, which the cluster's owner holds, and forge creates the build's least-privilege policy for that owner to attach. The first `fluid apply` started the diff --git a/fluid_build/iac/providers/aws.py b/fluid_build/iac/providers/aws.py index 90984c832..407b37f0a 100644 --- a/fluid_build/iac/providers/aws.py +++ b/fluid_build/iac/providers/aws.py @@ -1293,6 +1293,12 @@ def _emit_glue( # the renderer escapes at render time. Fallback is decided on the raw input # (``bucket_uses_fallback``), which a contract cannot spoof. bucket, path = _warehouse.normalize_location(loc, account_ref=_CALLER_ACCOUNT_TOKEN) + if is_iceberg_format(fmt): + # Iceberg keeps a table's location without the trailing slash + # (``LocationUtil.stripTrailingSlash``), and every commit writes it back into Glue + # (``GlueTableOperations``). With the slash here, each plan after a commit updated + # the table to put it back: an ``UpdateTable`` beside a running stream's commits. + path = path.rstrip("/") if _warehouse.bucket_uses_fallback(loc): storage["location"] = TofuExpr(f"s3://{bucket}/{_literal(path)}") else: diff --git a/fluid_build/providers/aws/plan/planner.py b/fluid_build/providers/aws/plan/planner.py index 4703718c3..645175779 100644 --- a/fluid_build/providers/aws/plan/planner.py +++ b/fluid_build/providers/aws/plan/planner.py @@ -631,7 +631,9 @@ def _plan_exposures( "database": database, "table": table, "columns": glue_columns, - "location": s3_location, + "location": _warehouse.iceberg_table_location( + location, account_ref=account_id + ), "icebergConfig": iceberg_config, "description": exposure.get("description"), } diff --git a/fluid_build/providers/aws/util/warehouse.py b/fluid_build/providers/aws/util/warehouse.py index 968daab3e..f82dc8206 100644 --- a/fluid_build/providers/aws/util/warehouse.py +++ b/fluid_build/providers/aws/util/warehouse.py @@ -80,6 +80,13 @@ def get_iceberg_warehouse(loc: Mapping[str, Any], *, account_ref: str) -> str: return f"s3://{bucket}/{path}" +def iceberg_table_location(loc: Mapping[str, Any], *, account_ref: str) -> str: + """An Iceberg table's location as Iceberg keeps it: :func:`get_iceberg_warehouse` + without its trailing slash (``LocationUtil.stripTrailingSlash``), which is what every + engine's commit writes back into the Glue table (``GlueTableOperations``).""" + return get_iceberg_warehouse(loc, account_ref=account_ref).rstrip("/") + + def bucket_uses_fallback(loc: Mapping[str, Any]) -> bool: """True when the bucket is absent / an unresolved ``{{ env.* }}`` template, so the warehouse falls back to the ``{account_ref}-fluid-data`` bucket. diff --git a/tests/iac/test_iac_aws_iceberg_catalog.py b/tests/iac/test_iac_aws_iceberg_catalog.py index b7a6c483e..da88b4ff3 100644 --- a/tests/iac/test_iac_aws_iceberg_catalog.py +++ b/tests/iac/test_iac_aws_iceberg_catalog.py @@ -177,7 +177,8 @@ def test_iceberg_table_format_spelling_is_the_same_table(self): "parameters": {"classification": "iceberg", "managed_by": "fluid"}, "storage_descriptor": { "columns": [{"name": "order_id", "type": "string"}, {"name": "qty", "type": "int"}], - "location": "s3://lake/orders/", + # Iceberg keeps its location without the trailing slash. + "location": "s3://lake/orders", }, "open_table_format_input": { "iceberg_input": {"metadata_operation": "CREATE", "version": "2"} diff --git a/tests/iac/test_iac_aws_iceberg_location_moto.py b/tests/iac/test_iac_aws_iceberg_location_moto.py new file mode 100644 index 000000000..a1f1db95b --- /dev/null +++ b/tests/iac/test_iac_aws_iceberg_location_moto.py @@ -0,0 +1,76 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""An Iceberg table's Glue location after an engine's commit plans no change, with the real +``tofu`` against moto. + +Iceberg keeps a table's location without the trailing slash, and every commit writes it back +into Glue (``GlueTableOperations``) beside the new ``metadata_location``. forge emitted the +location with the slash, so each plan after a commit updated the table to put the slash back: +an ``UpdateTable`` beside a running stream's commits. The commit is played here as Iceberg +makes it, through boto3. Skipped unless ``tofu`` is on PATH and moto's ``server`` extra is +installed; ``iac-tests.yml`` Stage 1 installs both and fails if this file only skipped. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +from fluid_build.cli import _apply_opentofu_engine as engine +from tests.iac.test_iac_aws_apply_engine_moto import ( # noqa: F401 - moto_endpoint is a fixture + _LOG, + _SKIP, + _SKIP_REASON, + _apply_args, + _aws_iceberg_contract, + _boto, + _drop_moto_provider_sidecar, + _write_plan, + moto_endpoint, +) + +pytestmark = [pytest.mark.integration, pytest.mark.provider, pytest.mark.aws] + + +def _commit_as_iceberg_does(glue, location: str) -> None: + """What ``GlueTableOperations.doCommit`` sends: the table as it is, its location as + Iceberg keeps it, and the new metadata file.""" + table = glue.get_table(DatabaseName="engine_silver", Name="events")["Table"] + storage = dict(table["StorageDescriptor"], Location=location) + parameters = dict( + table.get("Parameters") or {}, + metadata_location=f"{location}/metadata/00001-commit.metadata.json", + previous_metadata_location=f"{location}/metadata/00000-create.metadata.json", + ) + glue.update_table( + DatabaseName="engine_silver", + TableInput={ + "Name": "events", + "TableType": table.get("TableType", "EXTERNAL_TABLE"), + "StorageDescriptor": storage, + "Parameters": parameters, + }, + ) + + +@pytest.mark.skipif(_SKIP, reason=_SKIP_REASON) +def test_a_plan_after_an_iceberg_commit_changes_nothing( + moto_endpoint: str, tmp_path: Path, capsys +) -> None: + cid = "iac.engine.iceberg-location" + plan = _write_plan(tmp_path / "plan.json", _aws_iceberg_contract("iceberg-location-lake", cid)) + _drop_moto_provider_sidecar(tmp_path, cid, moto_endpoint) + assert engine.apply_via_opentofu(_apply_args(plan, tmp_path), _LOG) == 0 + + glue = _boto("glue", moto_endpoint) + emitted = glue.get_table(DatabaseName="engine_silver", Name="events")["Table"] + assert emitted["StorageDescriptor"]["Location"] == "s3://iceberg-location-lake/silver/events" + + _commit_as_iceberg_does(glue, "s3://iceberg-location-lake/silver/events") + capsys.readouterr() + assert engine.apply_via_opentofu(_apply_args(plan, tmp_path, dry_run=True), _LOG) == 0 + summary = re.search(r"tofu plan: \+(\d+) ~(\d+) -(\d+)", capsys.readouterr().out) + assert summary is not None and summary.groups() == ("0", "0", "0") diff --git a/tests/providers/aws/test_warehouse_derivation.py b/tests/providers/aws/test_warehouse_derivation.py index 89e062168..e15dc8c70 100644 --- a/tests/providers/aws/test_warehouse_derivation.py +++ b/tests/providers/aws/test_warehouse_derivation.py @@ -147,7 +147,7 @@ def test_explicit_bucket_byte_stable_no_extra_data_source(): plugin = get_iac_plugin("aws") tbl = next(iter(plugin.emit(contract)["aws_glue_catalog_table"].values())) loc = tbl["storage_descriptor"]["location"] - assert loc == "s3://lake/sales/orders/" + assert loc == "s3://lake/sales/orders" # as Iceberg keeps it: no trailing slash assert not isinstance(loc, TofuExpr) assert "aws_caller_identity" not in plugin.emit_data(contract) From fe5ad1281e8e8400da545826ee02dc75fb10804b Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 19 Nov 2026 07:09:19 +0100 Subject: [PATCH 2/3] test(iac): a table created before Iceberg metadata is updated in place The recorded state's table keeps the trailing slash forge emitted then, so the plan now updates its location once, in place, and replaces nothing: what the test is for. It asserts that the location is the only change and that every other resource is a no-op. --- tests/iac/test_iac_aws_iceberg_metadata.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/iac/test_iac_aws_iceberg_metadata.py b/tests/iac/test_iac_aws_iceberg_metadata.py index 06f9feeb4..8dec36c5e 100644 --- a/tests/iac/test_iac_aws_iceberg_metadata.py +++ b/tests/iac/test_iac_aws_iceberg_metadata.py @@ -140,8 +140,9 @@ def test_a_parquet_table_is_unchanged(contract): @pytest.mark.integration @pytest.mark.skipif(_TOFU is None, reason="tofu not on PATH") -def test_a_table_created_before_metadata_plans_no_change(tmp_path): - """No ForceNew replace, so the data-loss gate has nothing to stop.""" +def test_a_table_created_before_metadata_is_updated_in_place_not_replaced(tmp_path): + """No ForceNew replace, so the data-loss gate has nothing to stop. The one change is the + location, which loses its trailing slash as Iceberg keeps it, in place.""" from tests.iac.test_iac_tofu_validate import _tofu_init_or_skip plugin = get_iac_plugin("aws") @@ -179,6 +180,17 @@ def test_a_table_created_before_metadata_plans_no_change(tmp_path): table_address = ( "aws_glue_catalog_table.lane_sparkinfra_ice_upgrade_sparkinfra_ice_upgrade_events" ) - assert actions[table_address] == ["no-op"], actions - assert all(a == ["no-op"] for a in actions.values()), actions + assert actions[table_address] == ["update"], actions + change = next(rc["change"] for rc in plan["resource_changes"] if rc["address"] == table_address) + before = change["before"]["storage_descriptor"][0] + after = change["after"]["storage_descriptor"][0] + assert (before["location"], after["location"]) == ( + after["location"] + "/", + before["location"].rstrip("/"), + ) + assert {k: v for k, v in before.items() if k != "location"} == { + k: v for k, v in after.items() if k != "location" + } + others = {address: a for address, a in actions.items() if address != table_address} + assert all(a == ["no-op"] for a in others.values()), actions assert plugin.plan_data_changes(plan) == [] From 61b7368aeb323426c913b8519203e172fe85298d Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 19 Nov 2026 07:30:18 +0100 Subject: [PATCH 3/3] fix(iac): an Iceberg table at a bucket root keeps no trailing slash either From review: an Iceberg output with location.path "" still emitted s3://bucket/, so Iceberg's s3://bucket put the plan back to ~1. The location is built without the separator then, in the emit and (through iceberg_table_location) the planner alike, and a test pins both. The dead-letter test now also checks "beside, never inside" by whole path segments: order_lines__dead_letter starts with order_lines as a string, not as a directory. The dead letters' S3 prefix comes from the contract's path, not from this location, and keeps its slash. --- fluid_build/iac/providers/aws.py | 6 ++++-- tests/iac/test_iac_aws_spark_dead_letters.py | 5 +++++ tests/providers/aws/test_warehouse_derivation.py | 6 ++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/fluid_build/iac/providers/aws.py b/fluid_build/iac/providers/aws.py index 407b37f0a..90acce7ac 100644 --- a/fluid_build/iac/providers/aws.py +++ b/fluid_build/iac/providers/aws.py @@ -1299,10 +1299,12 @@ def _emit_glue( # (``GlueTableOperations``). With the slash here, each plan after a commit updated # the table to put it back: an ``UpdateTable`` beside a running stream's commits. path = path.rstrip("/") + # ``s3://bucket`` for an Iceberg table at a bucket's root, as Iceberg keeps that too. + sep = "/" if path or not is_iceberg_format(fmt) else "" if _warehouse.bucket_uses_fallback(loc): - storage["location"] = TofuExpr(f"s3://{bucket}/{_literal(path)}") + storage["location"] = TofuExpr(f"s3://{bucket}{sep}{_literal(path)}") else: - storage["location"] = f"s3://{bucket}/{path}" + storage["location"] = f"s3://{bucket}{sep}{path}" parameters: Dict[str, str] = {"classification": fmt, "managed_by": "fluid"} if "iceberg" in str(fmt).lower(): # AWS Glue / Athena identify an Iceberg table via this parameter. For a Glue- diff --git a/tests/iac/test_iac_aws_spark_dead_letters.py b/tests/iac/test_iac_aws_spark_dead_letters.py index 1604ccb10..377e1200e 100644 --- a/tests/iac/test_iac_aws_spark_dead_letters.py +++ b/tests/iac/test_iac_aws_spark_dead_letters.py @@ -70,6 +70,11 @@ def test_the_dead_letters_table_sits_beside_the_output_never_inside_it(contract) storage = table["storage_descriptor"] # As Iceberg keeps a table's location: no trailing slash (the S3 grant below has its own). assert storage["location"] == "s3://commerce-lake/iceberg/commerce/order_lines__dead_letter" + # Beside, never inside, by whole path segments (``order_lines__dead_letter`` starts with + # ``order_lines`` as a string, not as a directory). + output = resources["aws_glue_catalog_table"]["commerce_orders_stream_commerce_order_lines"] + outside = output["storage_descriptor"]["location"].rstrip("/") + "/" + assert not (storage["location"].rstrip("/") + "/").startswith(outside) assert {c["name"]: c["type"] for c in storage["columns"]} == { "error": "string", "raw_value": "binary", diff --git a/tests/providers/aws/test_warehouse_derivation.py b/tests/providers/aws/test_warehouse_derivation.py index e15dc8c70..571515751 100644 --- a/tests/providers/aws/test_warehouse_derivation.py +++ b/tests/providers/aws/test_warehouse_derivation.py @@ -152,6 +152,12 @@ def test_explicit_bucket_byte_stable_no_extra_data_source(): assert "aws_caller_identity" not in plugin.emit_data(contract) +def test_an_iceberg_table_at_a_bucket_root_has_no_trailing_slash_either(): + location = {"database": "sales", "table": "orders", "bucket": "lake", "path": ""} + assert _iac_location(location) == "s3://lake" + assert _planner_location(location) == "s3://lake" + + def test_same_warehouse_logical_equality(): from fluid_build.providers.aws.util.warehouse import same_warehouse