diff --git a/core/src/main/java/org/apache/iceberg/SerializableTable.java b/core/src/main/java/org/apache/iceberg/SerializableTable.java index a26fff1fd565..5b4cd0e55396 100644 --- a/core/src/main/java/org/apache/iceberg/SerializableTable.java +++ b/core/src/main/java/org/apache/iceberg/SerializableTable.java @@ -58,6 +58,7 @@ public class SerializableTable implements Table, HasTableOperations, Serializabl private final int defaultSpecId; private final Map specAsJsonMap; private final String sortOrderAsJson; + private final int defaultSortOrderId; private final Map sortOrderAsJsonMap; private final FileIO io; private final EncryptionManager encryption; @@ -83,6 +84,7 @@ protected SerializableTable(Table table) { Map specs = table.specs(); specs.forEach((specId, spec) -> specAsJsonMap.put(specId, PartitionSpecParser.toJson(spec))); this.sortOrderAsJson = SortOrderParser.toJson(table.sortOrder()); + this.defaultSortOrderId = table.sortOrder().orderId(); this.sortOrderAsJsonMap = Maps.newHashMap(); table .sortOrders() @@ -253,7 +255,8 @@ public Map sortOrders() { ImmutableMap.Builder sortOrders = ImmutableMap.builderWithExpectedSize(sortOrderAsJsonMap.size()); sortOrderAsJsonMap.forEach( - (id, json) -> sortOrders.put(id, SortOrderParser.fromJson(schema(), json))); + (id, json) -> + sortOrders.put(id, SortOrderParser.fromJson(schema(), json, defaultSortOrderId))); this.lazySortOrders = sortOrders.build(); } else if (lazySortOrders == null) { this.lazySortOrders = lazyTable.sortOrders(); diff --git a/core/src/main/java/org/apache/iceberg/SortOrderParser.java b/core/src/main/java/org/apache/iceberg/SortOrderParser.java index 31307cf9dc7f..53d7e5090c76 100644 --- a/core/src/main/java/org/apache/iceberg/SortOrderParser.java +++ b/core/src/main/java/org/apache/iceberg/SortOrderParser.java @@ -112,6 +112,10 @@ public static SortOrder fromJson(Schema schema, String json) { return fromJson(json).bind(schema); } + public static SortOrder fromJson(Schema schema, String json, int defaultSortOrderId) { + return JsonUtil.parse(json, node -> fromJson(schema, node, defaultSortOrderId)); + } + public static SortOrder fromJson(Schema schema, JsonNode json, int defaultSortOrderId) { UnboundSortOrder unboundSortOrder = fromJson(json); diff --git a/core/src/test/java/org/apache/iceberg/hadoop/TestTableSerialization.java b/core/src/test/java/org/apache/iceberg/hadoop/TestTableSerialization.java index 5103e2e9be92..ece9b24af3d1 100644 --- a/core/src/test/java/org/apache/iceberg/hadoop/TestTableSerialization.java +++ b/core/src/test/java/org/apache/iceberg/hadoop/TestTableSerialization.java @@ -79,6 +79,22 @@ public void testSerializableTable() throws IOException, ClassNotFoundException { .isEqualTo(TableUtil.metadataFileLocation(table)); } + @Test + public void testSerializableTableSortOrdersWithDroppedColumn() + throws IOException, ClassNotFoundException { + table.updateSchema().addColumn("ts", Types.LongType.get()).commit(); + table.replaceSortOrder().asc("id").asc("ts").commit(); + + // historical sort order 1 still references "ts" after the column is dropped + table.replaceSortOrder().asc("id").commit(); + table.updateSchema().deleteColumn("ts").commit(); + + TestHelpers.assertSerializedAndLoadedMetadata(table, TestHelpers.roundTripSerialize(table)); + Table serializableTable = SerializableTable.copyOf(table); + TestHelpers.assertSerializedAndLoadedMetadata( + serializableTable, TestHelpers.KryoHelpers.roundTripSerialize(serializableTable)); + } + @Test public void testSerializableTableWithSnapshot() throws IOException, ClassNotFoundException { table.newAppend().appendFile(FILE_A).commit();