diff --git a/.github/workflows/iac-tests.yml b/.github/workflows/iac-tests.yml index a906a7d72..de0533a1f 100644 --- a/.github/workflows/iac-tests.yml +++ b/.github/workflows/iac-tests.yml @@ -205,13 +205,15 @@ jobs: # A Glue table's partition keys and Athena projection: whether a changed # layout replaces a live table, and dropping the keys updates it in place, # depends on how `replace_triggered_by` meets the trigger's create, update - # and destroy, so only a real plan against an applied table proves it. + # and destroy, so only a real plan against an applied table proves it. So + # does an Iceberg table's plan after an engine's commit rewrote its location. - name: "Glue partition keys apply + plan (tofu vs moto, creds-free)" - tests: tests/iac/test_iac_aws_partition_plan_moto.py + tests: tests/iac/test_iac_aws_partition_plan_moto.py tests/iac/test_iac_aws_iceberg_location_moto.py pytest-args: "" junit: glue-partition-moto.xml require: | Glue partition moto=tests/iac/test_iac_aws_partition_plan_moto.py + Iceberg location moto=tests/iac/test_iac_aws_iceberg_location_moto.py # `fluid diff` refreshes the apply's OpenTofu state: a real apply, # then the refresh plan, against moto. Whether a clean apply reads as # no drift depends on what the provider reads back, so only a real diff --git a/.secrets.baseline b/.secrets.baseline index f2d8c625e..35b71b2d5 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -139,7 +139,7 @@ "filename": ".github/workflows/iac-tests.yml", "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", "is_verified": false, - "line_number": 441 + "line_number": 443 } ], ".github/workflows/integration.yml": [ diff --git a/CHANGELOG.md b/CHANGELOG.md index 9809e02a0..8d034bb48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -99,6 +99,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 The artifact bucket's write access stays the boundary. A scheduler's role already reads both objects. The runner changed, so each running Spark stream restarts once on its next apply. +- **An Iceberg table on Glue plans no change after an engine commits to it.** forge emitted the + table's location with a trailing slash; Iceberg keeps it without one and writes it back into + Glue on every commit. So each `fluid apply` after a commit (every scheduled run beside a + stream) planned `~1` per written table and sent an `UpdateTable` while the stream kept + committing, with `previous_metadata_location` from the plan's refresh. The location is now + emitted as Iceberg keeps it. The first apply after the upgrade updates each existing Iceberg + table's location once, in place; Hive-format tables keep their trailing slash. + ## [0.33.0] — 2026-11-19 ### Added diff --git a/fluid_build/iac/providers/aws.py b/fluid_build/iac/providers/aws.py index 8d65ab155..19de0c0c7 100644 --- a/fluid_build/iac/providers/aws.py +++ b/fluid_build/iac/providers/aws.py @@ -1316,10 +1316,18 @@ def _emit_glue( # the renderer escapes at render time. Fallback is decided on the raw input # (``bucket_uses_fallback``), which a contract cannot spoof. bucket, path = _warehouse.normalize_location(loc, account_ref=_CALLER_ACCOUNT_TOKEN) + if is_iceberg_format(fmt): + # Iceberg keeps a table's location without the trailing slash + # (``LocationUtil.stripTrailingSlash``), and every commit writes it back into Glue + # (``GlueTableOperations``). With the slash here, each plan after a commit updated + # the table to put it back: an ``UpdateTable`` beside a running stream's commits. + path = path.rstrip("/") + # ``s3://bucket`` for an Iceberg table at a bucket's root, as Iceberg keeps that too. + sep = "/" if path or not is_iceberg_format(fmt) else "" if _warehouse.bucket_uses_fallback(loc): - storage["location"] = TofuExpr(f"s3://{bucket}/{_literal(path)}") + storage["location"] = TofuExpr(f"s3://{bucket}{sep}{_literal(path)}") else: - storage["location"] = f"s3://{bucket}/{path}" + storage["location"] = f"s3://{bucket}{sep}{path}" parameters: Dict[str, str] = {"classification": fmt, "managed_by": "fluid"} if "iceberg" in str(fmt).lower(): # AWS Glue / Athena identify an Iceberg table via this parameter. For a Glue- diff --git a/fluid_build/providers/aws/plan/planner.py b/fluid_build/providers/aws/plan/planner.py index 4703718c3..645175779 100644 --- a/fluid_build/providers/aws/plan/planner.py +++ b/fluid_build/providers/aws/plan/planner.py @@ -631,7 +631,9 @@ def _plan_exposures( "database": database, "table": table, "columns": glue_columns, - "location": s3_location, + "location": _warehouse.iceberg_table_location( + location, account_ref=account_id + ), "icebergConfig": iceberg_config, "description": exposure.get("description"), } diff --git a/fluid_build/providers/aws/util/warehouse.py b/fluid_build/providers/aws/util/warehouse.py index 968daab3e..f82dc8206 100644 --- a/fluid_build/providers/aws/util/warehouse.py +++ b/fluid_build/providers/aws/util/warehouse.py @@ -80,6 +80,13 @@ def get_iceberg_warehouse(loc: Mapping[str, Any], *, account_ref: str) -> str: return f"s3://{bucket}/{path}" +def iceberg_table_location(loc: Mapping[str, Any], *, account_ref: str) -> str: + """An Iceberg table's location as Iceberg keeps it: :func:`get_iceberg_warehouse` + without its trailing slash (``LocationUtil.stripTrailingSlash``), which is what every + engine's commit writes back into the Glue table (``GlueTableOperations``).""" + return get_iceberg_warehouse(loc, account_ref=account_ref).rstrip("/") + + def bucket_uses_fallback(loc: Mapping[str, Any]) -> bool: """True when the bucket is absent / an unresolved ``{{ env.* }}`` template, so the warehouse falls back to the ``{account_ref}-fluid-data`` bucket. diff --git a/tests/iac/test_iac_aws_iceberg_catalog.py b/tests/iac/test_iac_aws_iceberg_catalog.py index b7a6c483e..da88b4ff3 100644 --- a/tests/iac/test_iac_aws_iceberg_catalog.py +++ b/tests/iac/test_iac_aws_iceberg_catalog.py @@ -177,7 +177,8 @@ def test_iceberg_table_format_spelling_is_the_same_table(self): "parameters": {"classification": "iceberg", "managed_by": "fluid"}, "storage_descriptor": { "columns": [{"name": "order_id", "type": "string"}, {"name": "qty", "type": "int"}], - "location": "s3://lake/orders/", + # Iceberg keeps its location without the trailing slash. + "location": "s3://lake/orders", }, "open_table_format_input": { "iceberg_input": {"metadata_operation": "CREATE", "version": "2"} diff --git a/tests/iac/test_iac_aws_iceberg_location_moto.py b/tests/iac/test_iac_aws_iceberg_location_moto.py new file mode 100644 index 000000000..a1f1db95b --- /dev/null +++ b/tests/iac/test_iac_aws_iceberg_location_moto.py @@ -0,0 +1,76 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""An Iceberg table's Glue location after an engine's commit plans no change, with the real +``tofu`` against moto. + +Iceberg keeps a table's location without the trailing slash, and every commit writes it back +into Glue (``GlueTableOperations``) beside the new ``metadata_location``. forge emitted the +location with the slash, so each plan after a commit updated the table to put the slash back: +an ``UpdateTable`` beside a running stream's commits. The commit is played here as Iceberg +makes it, through boto3. Skipped unless ``tofu`` is on PATH and moto's ``server`` extra is +installed; ``iac-tests.yml`` Stage 1 installs both and fails if this file only skipped. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +from fluid_build.cli import _apply_opentofu_engine as engine +from tests.iac.test_iac_aws_apply_engine_moto import ( # noqa: F401 - moto_endpoint is a fixture + _LOG, + _SKIP, + _SKIP_REASON, + _apply_args, + _aws_iceberg_contract, + _boto, + _drop_moto_provider_sidecar, + _write_plan, + moto_endpoint, +) + +pytestmark = [pytest.mark.integration, pytest.mark.provider, pytest.mark.aws] + + +def _commit_as_iceberg_does(glue, location: str) -> None: + """What ``GlueTableOperations.doCommit`` sends: the table as it is, its location as + Iceberg keeps it, and the new metadata file.""" + table = glue.get_table(DatabaseName="engine_silver", Name="events")["Table"] + storage = dict(table["StorageDescriptor"], Location=location) + parameters = dict( + table.get("Parameters") or {}, + metadata_location=f"{location}/metadata/00001-commit.metadata.json", + previous_metadata_location=f"{location}/metadata/00000-create.metadata.json", + ) + glue.update_table( + DatabaseName="engine_silver", + TableInput={ + "Name": "events", + "TableType": table.get("TableType", "EXTERNAL_TABLE"), + "StorageDescriptor": storage, + "Parameters": parameters, + }, + ) + + +@pytest.mark.skipif(_SKIP, reason=_SKIP_REASON) +def test_a_plan_after_an_iceberg_commit_changes_nothing( + moto_endpoint: str, tmp_path: Path, capsys +) -> None: + cid = "iac.engine.iceberg-location" + plan = _write_plan(tmp_path / "plan.json", _aws_iceberg_contract("iceberg-location-lake", cid)) + _drop_moto_provider_sidecar(tmp_path, cid, moto_endpoint) + assert engine.apply_via_opentofu(_apply_args(plan, tmp_path), _LOG) == 0 + + glue = _boto("glue", moto_endpoint) + emitted = glue.get_table(DatabaseName="engine_silver", Name="events")["Table"] + assert emitted["StorageDescriptor"]["Location"] == "s3://iceberg-location-lake/silver/events" + + _commit_as_iceberg_does(glue, "s3://iceberg-location-lake/silver/events") + capsys.readouterr() + assert engine.apply_via_opentofu(_apply_args(plan, tmp_path, dry_run=True), _LOG) == 0 + summary = re.search(r"tofu plan: \+(\d+) ~(\d+) -(\d+)", capsys.readouterr().out) + assert summary is not None and summary.groups() == ("0", "0", "0") diff --git a/tests/iac/test_iac_aws_iceberg_metadata.py b/tests/iac/test_iac_aws_iceberg_metadata.py index 06f9feeb4..8dec36c5e 100644 --- a/tests/iac/test_iac_aws_iceberg_metadata.py +++ b/tests/iac/test_iac_aws_iceberg_metadata.py @@ -140,8 +140,9 @@ def test_a_parquet_table_is_unchanged(contract): @pytest.mark.integration @pytest.mark.skipif(_TOFU is None, reason="tofu not on PATH") -def test_a_table_created_before_metadata_plans_no_change(tmp_path): - """No ForceNew replace, so the data-loss gate has nothing to stop.""" +def test_a_table_created_before_metadata_is_updated_in_place_not_replaced(tmp_path): + """No ForceNew replace, so the data-loss gate has nothing to stop. The one change is the + location, which loses its trailing slash as Iceberg keeps it, in place.""" from tests.iac.test_iac_tofu_validate import _tofu_init_or_skip plugin = get_iac_plugin("aws") @@ -179,6 +180,17 @@ def test_a_table_created_before_metadata_plans_no_change(tmp_path): table_address = ( "aws_glue_catalog_table.lane_sparkinfra_ice_upgrade_sparkinfra_ice_upgrade_events" ) - assert actions[table_address] == ["no-op"], actions - assert all(a == ["no-op"] for a in actions.values()), actions + assert actions[table_address] == ["update"], actions + change = next(rc["change"] for rc in plan["resource_changes"] if rc["address"] == table_address) + before = change["before"]["storage_descriptor"][0] + after = change["after"]["storage_descriptor"][0] + assert (before["location"], after["location"]) == ( + after["location"] + "/", + before["location"].rstrip("/"), + ) + assert {k: v for k, v in before.items() if k != "location"} == { + k: v for k, v in after.items() if k != "location" + } + others = {address: a for address, a in actions.items() if address != table_address} + assert all(a == ["no-op"] for a in others.values()), actions assert plugin.plan_data_changes(plan) == [] diff --git a/tests/iac/test_iac_aws_spark_dead_letters.py b/tests/iac/test_iac_aws_spark_dead_letters.py index ef5c2583b..8eca1ec0d 100644 --- a/tests/iac/test_iac_aws_spark_dead_letters.py +++ b/tests/iac/test_iac_aws_spark_dead_letters.py @@ -68,7 +68,13 @@ def test_the_dead_letters_table_sits_beside_the_output_never_inside_it(contract) table = resources["aws_glue_catalog_table"][TABLE_KEY] assert table["name"] == "order_lines__dead_letter" storage = table["storage_descriptor"] - assert storage["location"] == "s3://commerce-lake/iceberg/commerce/order_lines__dead_letter/" + # As Iceberg keeps a table's location: no trailing slash (the S3 grant below has its own). + assert storage["location"] == "s3://commerce-lake/iceberg/commerce/order_lines__dead_letter" + # Beside, never inside, by whole path segments (``order_lines__dead_letter`` starts with + # ``order_lines`` as a string, not as a directory). + output = resources["aws_glue_catalog_table"]["commerce_orders_stream_commerce_order_lines"] + outside = output["storage_descriptor"]["location"].rstrip("/") + "/" + assert not (storage["location"].rstrip("/") + "/").startswith(outside) assert {c["name"]: c["type"] for c in storage["columns"]} == { "error": "string", "raw_value": "binary", @@ -107,7 +113,7 @@ def test_the_dead_letters_table_sits_beside_the_output_never_inside_it(contract) def test_a_named_dead_letters_table_is_the_one_emitted_and_granted(contract): resources, data = emit(with_dead_letters(contract, table="order_rejects")) table = resources["aws_glue_catalog_table"]["commerce_orders_stream_commerce_order_rejects"] - assert table["storage_descriptor"]["location"].endswith("/iceberg/commerce/order_rejects/") + assert table["storage_descriptor"]["location"].endswith("/iceberg/commerce/order_rejects") glue = statement(data["aws_iam_policy_document"][f"{KEY}_access"], "GlueTable") assert glue["resources"][-1].endswith(":table/commerce/order_rejects") diff --git a/tests/providers/aws/test_warehouse_derivation.py b/tests/providers/aws/test_warehouse_derivation.py index 89e062168..571515751 100644 --- a/tests/providers/aws/test_warehouse_derivation.py +++ b/tests/providers/aws/test_warehouse_derivation.py @@ -147,11 +147,17 @@ def test_explicit_bucket_byte_stable_no_extra_data_source(): plugin = get_iac_plugin("aws") tbl = next(iter(plugin.emit(contract)["aws_glue_catalog_table"].values())) loc = tbl["storage_descriptor"]["location"] - assert loc == "s3://lake/sales/orders/" + assert loc == "s3://lake/sales/orders" # as Iceberg keeps it: no trailing slash assert not isinstance(loc, TofuExpr) assert "aws_caller_identity" not in plugin.emit_data(contract) +def test_an_iceberg_table_at_a_bucket_root_has_no_trailing_slash_either(): + location = {"database": "sales", "table": "orders", "bucket": "lake", "path": ""} + assert _iac_location(location) == "s3://lake" + assert _planner_location(location) == "s3://lake" + + def test_same_warehouse_logical_equality(): from fluid_build.providers.aws.util.warehouse import same_warehouse