From a64c53319068abeb3573fdfb35034e7e87324606 Mon Sep 17 00:00:00 2001 From: Yong Date: Thu, 21 May 2026 19:02:48 -0500 Subject: [PATCH 1/2] Fix issue 16519 --- .../org/apache/iceberg/SerializableTable.java | 5 ++++- .../org/apache/iceberg/SortOrderParser.java | 4 ++++ .../hadoop/TestTableSerialization.java | 21 +++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/iceberg/SerializableTable.java b/core/src/main/java/org/apache/iceberg/SerializableTable.java index a26fff1fd565..5b4cd0e55396 100644 --- a/core/src/main/java/org/apache/iceberg/SerializableTable.java +++ b/core/src/main/java/org/apache/iceberg/SerializableTable.java @@ -58,6 +58,7 @@ public class SerializableTable implements Table, HasTableOperations, Serializabl private final int defaultSpecId; private final Map specAsJsonMap; private final String sortOrderAsJson; + private final int defaultSortOrderId; private final Map sortOrderAsJsonMap; private final FileIO io; private final EncryptionManager encryption; @@ -83,6 +84,7 @@ protected SerializableTable(Table table) { Map specs = table.specs(); specs.forEach((specId, spec) -> specAsJsonMap.put(specId, PartitionSpecParser.toJson(spec))); this.sortOrderAsJson = SortOrderParser.toJson(table.sortOrder()); + this.defaultSortOrderId = table.sortOrder().orderId(); this.sortOrderAsJsonMap = Maps.newHashMap(); table .sortOrders() @@ -253,7 +255,8 @@ public Map sortOrders() { ImmutableMap.Builder sortOrders = ImmutableMap.builderWithExpectedSize(sortOrderAsJsonMap.size()); sortOrderAsJsonMap.forEach( - (id, json) -> sortOrders.put(id, SortOrderParser.fromJson(schema(), json))); + (id, json) -> + sortOrders.put(id, SortOrderParser.fromJson(schema(), json, defaultSortOrderId))); this.lazySortOrders = sortOrders.build(); } else if (lazySortOrders == null) { this.lazySortOrders = lazyTable.sortOrders(); diff --git a/core/src/main/java/org/apache/iceberg/SortOrderParser.java b/core/src/main/java/org/apache/iceberg/SortOrderParser.java index 31307cf9dc7f..0a675a2a9337 100644 --- a/core/src/main/java/org/apache/iceberg/SortOrderParser.java +++ b/core/src/main/java/org/apache/iceberg/SortOrderParser.java @@ -112,6 +112,10 @@ public static SortOrder fromJson(Schema schema, String json) { return fromJson(json).bind(schema); } + public static SortOrder fromJson(Schema schema, String json, int defaultSortOderId) { + return JsonUtil.parse(json, node -> fromJson(schema, node, defaultSortOderId)); + } + public static SortOrder fromJson(Schema schema, JsonNode json, int defaultSortOrderId) { UnboundSortOrder unboundSortOrder = fromJson(json); diff --git a/core/src/test/java/org/apache/iceberg/hadoop/TestTableSerialization.java b/core/src/test/java/org/apache/iceberg/hadoop/TestTableSerialization.java index 5103e2e9be92..95723ca62191 100644 --- a/core/src/test/java/org/apache/iceberg/hadoop/TestTableSerialization.java +++ b/core/src/test/java/org/apache/iceberg/hadoop/TestTableSerialization.java @@ -79,6 +79,27 @@ public void testSerializableTable() throws IOException, ClassNotFoundException { .isEqualTo(TableUtil.metadataFileLocation(table)); } + @Test + public void testSerializableTableSortOrdersWithDroppedColumn() { + // add an extra column and establish sort order 1 on id + ts + table.updateSchema().addColumn("ts", Types.LongType.get()).commit(); + table.replaceSortOrder().asc("id").asc("ts").commit(); + table.newAppend().appendFile(FILE_A).commit(); + + // switch to sort order 2 using only "id", then drop "ts" + // historical sort order 1 still references the dropped "ts" field + table.replaceSortOrder().asc("id").commit(); + table.updateSchema().deleteColumn("ts").commit(); + + // SerializableTable.copyOf captures all sort orders at construction time + Table serialized = SerializableTable.copyOf(table); + + // sortOrders() must return all historical orders without throwing a ValidationException, + // even though sort order 1 references the now-dropped "ts" field + assertThat(serialized.sortOrders()).hasSize(3); // unsorted(0), order-1(id+ts), order-2(id) + assertThat(serialized.sortOrder().fields()).hasSize(1); // current default is order-2 + } + @Test public void testSerializableTableWithSnapshot() throws IOException, ClassNotFoundException { table.newAppend().appendFile(FILE_A).commit(); From 50136e9a9422d85ae82fc23ac54dc8b8d774b109 Mon Sep 17 00:00:00 2001 From: Yong Date: Thu, 21 May 2026 20:44:36 -0500 Subject: [PATCH 2/2] Fix typo and formalize the test case --- .../org/apache/iceberg/SortOrderParser.java | 4 ++-- .../hadoop/TestTableSerialization.java | 19 +++++++------------ 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/SortOrderParser.java b/core/src/main/java/org/apache/iceberg/SortOrderParser.java index 0a675a2a9337..53d7e5090c76 100644 --- a/core/src/main/java/org/apache/iceberg/SortOrderParser.java +++ b/core/src/main/java/org/apache/iceberg/SortOrderParser.java @@ -112,8 +112,8 @@ public static SortOrder fromJson(Schema schema, String json) { return fromJson(json).bind(schema); } - public static SortOrder fromJson(Schema schema, String json, int defaultSortOderId) { - return JsonUtil.parse(json, node -> fromJson(schema, node, defaultSortOderId)); + public static SortOrder fromJson(Schema schema, String json, int defaultSortOrderId) { + return JsonUtil.parse(json, node -> fromJson(schema, node, defaultSortOrderId)); } public static SortOrder fromJson(Schema schema, JsonNode json, int defaultSortOrderId) { diff --git a/core/src/test/java/org/apache/iceberg/hadoop/TestTableSerialization.java b/core/src/test/java/org/apache/iceberg/hadoop/TestTableSerialization.java index 95723ca62191..ece9b24af3d1 100644 --- a/core/src/test/java/org/apache/iceberg/hadoop/TestTableSerialization.java +++ b/core/src/test/java/org/apache/iceberg/hadoop/TestTableSerialization.java @@ -80,24 +80,19 @@ public void testSerializableTable() throws IOException, ClassNotFoundException { } @Test - public void testSerializableTableSortOrdersWithDroppedColumn() { - // add an extra column and establish sort order 1 on id + ts + public void testSerializableTableSortOrdersWithDroppedColumn() + throws IOException, ClassNotFoundException { table.updateSchema().addColumn("ts", Types.LongType.get()).commit(); table.replaceSortOrder().asc("id").asc("ts").commit(); - table.newAppend().appendFile(FILE_A).commit(); - // switch to sort order 2 using only "id", then drop "ts" - // historical sort order 1 still references the dropped "ts" field + // historical sort order 1 still references "ts" after the column is dropped table.replaceSortOrder().asc("id").commit(); table.updateSchema().deleteColumn("ts").commit(); - // SerializableTable.copyOf captures all sort orders at construction time - Table serialized = SerializableTable.copyOf(table); - - // sortOrders() must return all historical orders without throwing a ValidationException, - // even though sort order 1 references the now-dropped "ts" field - assertThat(serialized.sortOrders()).hasSize(3); // unsorted(0), order-1(id+ts), order-2(id) - assertThat(serialized.sortOrder().fields()).hasSize(1); // current default is order-2 + TestHelpers.assertSerializedAndLoadedMetadata(table, TestHelpers.roundTripSerialize(table)); + Table serializableTable = SerializableTable.copyOf(table); + TestHelpers.assertSerializedAndLoadedMetadata( + serializableTable, TestHelpers.KryoHelpers.roundTripSerialize(serializableTable)); } @Test