From dcdacd72b0ce1b50ab990c91e9a832e284b66beb Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Wed, 11 Nov 2026 15:12:16 -0800 Subject: [PATCH 1/4] API, Core: Add Snapshot.rootManifestLocation() for v4 root manifests v4 snapshot metadata has no format-version field and stores the snapshot's root file in root-manifest instead of manifest-list. Add Snapshot.rootManifestLocation() and keep manifestListLocation() for v1-v3 snapshots. Each snapshot sets at most one of the two. Commits still write a manifest list. Readers that open the snapshot file as a manifest list stay on manifestListLocation() until a root manifest is written. Generated-by: Cursor Grok 4.7 Co-authored-by: Cursor --- .../java/org/apache/iceberg/Snapshot.java | 18 ++- .../java/org/apache/iceberg/BaseSnapshot.java | 20 ++- .../apache/iceberg/RewriteTablePathUtil.java | 7 +- .../org/apache/iceberg/SnapshotParser.java | 15 +- .../org/apache/iceberg/SnapshotProducer.java | 1 + .../apache/iceberg/TestDataTaskParser.java | 2 + .../iceberg/TestMetadataUpdateParser.java | 35 +++++ .../iceberg/TestRowLineageMetadata.java | 134 ++++++++++++++++-- .../java/org/apache/iceberg/TestSnapshot.java | 78 +++++++++- .../org/apache/iceberg/TestSnapshotJson.java | 44 ++++++ .../org/apache/iceberg/TestTableMetadata.java | 75 +++++++++- 11 files changed, 400 insertions(+), 29 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/Snapshot.java b/api/src/main/java/org/apache/iceberg/Snapshot.java index 8a74dca6d053..28efa6255938 100644 --- a/api/src/main/java/org/apache/iceberg/Snapshot.java +++ b/api/src/main/java/org/apache/iceberg/Snapshot.java @@ -166,10 +166,26 @@ default Iterable removedDeleteFiles(FileIO io) { /** * Return the location of this snapshot's manifest list, or null if it is not separate. * - * @return the location of the manifest list for this Snapshot + *

Only set for tables with format version 1 through 3. Snapshots in v4 tables use {@link + * #rootManifestLocation()} instead. + * + * @return the location of the manifest list for this snapshot */ String manifestListLocation(); + /** + * Return the location of this snapshot's root manifest, or null if this snapshot has no root + * manifest. + * + *

v4 snapshots store this in {@code root-manifest} instead of a manifest list. At most one of + * this and {@link #manifestListLocation()} is set. + * + * @return the location of the root manifest for this snapshot + */ + default String rootManifestLocation() { + return null; + } + /** * Return the id of the schema used when this snapshot was created, or null if this information is * not available. diff --git a/core/src/main/java/org/apache/iceberg/BaseSnapshot.java b/core/src/main/java/org/apache/iceberg/BaseSnapshot.java index 826b9624c0e6..aa079fc1521b 100644 --- a/core/src/main/java/org/apache/iceberg/BaseSnapshot.java +++ b/core/src/main/java/org/apache/iceberg/BaseSnapshot.java @@ -39,6 +39,7 @@ class BaseSnapshot implements Snapshot { private final long sequenceNumber; private final long timestampMillis; private final String manifestListLocation; + private final String rootManifestLocation; private final String operation; private final Map summary; private final Integer schemaId; @@ -65,9 +66,14 @@ class BaseSnapshot implements Snapshot { Map summary, Integer schemaId, String manifestList, + String rootManifest, Long firstRowId, Long addedRows, String keyId) { + Preconditions.checkArgument( + manifestList == null || rootManifest == null, + "Snapshot %s cannot have both a manifest list and a root manifest", + snapshotId); Preconditions.checkArgument( firstRowId == null || firstRowId >= 0, "Invalid first-row-id (cannot be negative): %s", @@ -87,6 +93,7 @@ class BaseSnapshot implements Snapshot { this.summary = summary; this.schemaId = schemaId; this.manifestListLocation = manifestList; + this.rootManifestLocation = rootManifest; this.v1ManifestLocations = null; this.firstRowId = firstRowId; this.addedRows = firstRowId != null ? addedRows : null; @@ -110,6 +117,7 @@ class BaseSnapshot implements Snapshot { this.summary = summary; this.schemaId = schemaId; this.manifestListLocation = null; + this.rootManifestLocation = null; this.v1ManifestLocations = v1ManifestLocations; this.firstRowId = null; this.addedRows = null; @@ -181,7 +189,11 @@ private void cacheManifests(FileIO fileIO) { } if (allManifests == null) { - // if manifests isn't set, then the snapshotFile is set and should be read to get the list + // if manifests isn't set, then the manifest list must be read + Preconditions.checkArgument( + manifestListLocation != null, + "Cannot read manifests from snapshot %s: no manifest list", + snapshotId); this.allManifests = ManifestLists.read( ManifestLists.newInputFile( @@ -261,6 +273,11 @@ public String manifestListLocation() { return manifestListLocation; } + @Override + public String rootManifestLocation() { + return rootManifestLocation; + } + private void cacheDeleteFileChanges(FileIO fileIO) { Preconditions.checkArgument(fileIO != null, "Cannot cache delete file changes: FileIO is null"); @@ -370,6 +387,7 @@ public String toString() { .add("operation", operation) .add("summary", summary) .add("manifest-list", manifestListLocation) + .add("root-manifest", rootManifestLocation) .add("schema-id", schemaId) .add("first-row-id", firstRowId) .add("added-rows", addedRows) diff --git a/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java b/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java index e1afe80d661c..ef295bed51be 100644 --- a/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java +++ b/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java @@ -241,8 +241,8 @@ private static List updatePathInSnapshots( TableMetadata metadata, String sourcePrefix, String targetPrefix) { List newSnapshots = Lists.newArrayListWithCapacity(metadata.snapshots().size()); for (Snapshot snapshot : metadata.snapshots()) { - String newManifestListLocation = - newPath(snapshot.manifestListLocation(), sourcePrefix, targetPrefix); + String manifestList = snapshot.manifestListLocation(); + String rootManifest = snapshot.rootManifestLocation(); Snapshot newSnapshot = new BaseSnapshot( snapshot.sequenceNumber(), @@ -252,7 +252,8 @@ private static List updatePathInSnapshots( snapshot.operation(), snapshot.summary(), snapshot.schemaId(), - newManifestListLocation, + manifestList == null ? null : newPath(manifestList, sourcePrefix, targetPrefix), + rootManifest == null ? null : newPath(rootManifest, sourcePrefix, targetPrefix), snapshot.firstRowId(), snapshot.addedRows(), snapshot.keyId()); diff --git a/core/src/main/java/org/apache/iceberg/SnapshotParser.java b/core/src/main/java/org/apache/iceberg/SnapshotParser.java index 53cec16dcd87..fd896f1df5a5 100644 --- a/core/src/main/java/org/apache/iceberg/SnapshotParser.java +++ b/core/src/main/java/org/apache/iceberg/SnapshotParser.java @@ -50,6 +50,7 @@ private SnapshotParser() {} private static final String OPERATION = "operation"; private static final String MANIFESTS = "manifests"; private static final String MANIFEST_LIST = "manifest-list"; + private static final String ROOT_MANIFEST = "root-manifest"; private static final String SCHEMA_ID = "schema-id"; private static final String FIRST_ROW_ID = "first-row-id"; private static final String ADDED_ROWS = "added-rows"; @@ -83,7 +84,10 @@ static void toJson(Snapshot snapshot, JsonGenerator generator) throws IOExceptio } String manifestList = snapshot.manifestListLocation(); - if (manifestList != null) { + String rootManifest = snapshot.rootManifestLocation(); + if (rootManifest != null) { + generator.writeStringField(ROOT_MANIFEST, rootManifest); + } else if (manifestList != null) { // write just the location. manifests should not be embedded in JSON along with a list generator.writeStringField(MANIFEST_LIST, manifestList); } else { @@ -176,9 +180,11 @@ static Snapshot fromJson(JsonNode node) { String keyId = JsonUtil.getStringOrNull(KEY_ID, node); - if (node.has(MANIFEST_LIST)) { - // the manifest list is stored in a manifest list file - String manifestList = JsonUtil.getString(MANIFEST_LIST, node); + String manifestList = JsonUtil.getStringOrNull(MANIFEST_LIST, node); + String rootManifest = JsonUtil.getStringOrNull(ROOT_MANIFEST, node); + + if (manifestList != null || rootManifest != null) { + // manifest-list is used by v1-v3 snapshots, root-manifest by v4 snapshots return new BaseSnapshot( sequenceNumber, snapshotId, @@ -188,6 +194,7 @@ static Snapshot fromJson(JsonNode node) { summary, schemaId, manifestList, + rootManifest, firstRowId, addedRows, keyId); diff --git a/core/src/main/java/org/apache/iceberg/SnapshotProducer.java b/core/src/main/java/org/apache/iceberg/SnapshotProducer.java index 93133d2fe56e..48f5351e44ee 100644 --- a/core/src/main/java/org/apache/iceberg/SnapshotProducer.java +++ b/core/src/main/java/org/apache/iceberg/SnapshotProducer.java @@ -372,6 +372,7 @@ public Snapshot apply() { summary(base), base.currentSchemaId(), manifestList.location(), + null /* rootManifest */, nextRowId, assignedRows, manifestListEncryptionKeys != null ? manifestListEncryptionKeys.fileKey().keyId() : null); diff --git a/core/src/test/java/org/apache/iceberg/TestDataTaskParser.java b/core/src/test/java/org/apache/iceberg/TestDataTaskParser.java index b85d14525b68..8e8361e68f43 100644 --- a/core/src/test/java/org/apache/iceberg/TestDataTaskParser.java +++ b/core/src/test/java/org/apache/iceberg/TestDataTaskParser.java @@ -219,6 +219,7 @@ private DataTask createDataTask() { summary1, 1, "file:/tmp/manifest1.avro", + null /* rootManifest */, null, null, null), @@ -231,6 +232,7 @@ private DataTask createDataTask() { summary2, 1, "file:/tmp/manifest2.avro", + null /* rootManifest */, null, null, null)); diff --git a/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java b/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java index c661ac834d45..9665c53adb1f 100644 --- a/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java +++ b/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java @@ -381,6 +381,7 @@ public void testAddSnapshotToJson() throws IOException { ImmutableMap.of("files-added", "4", "files-deleted", "100"), schemaId, manifestList, + null /* rootManifest */, firstRowId, addedRows, "key-1"); @@ -414,6 +415,7 @@ public void testAddSnapshotFromJson() throws IOException { summary, schemaId, manifestList, + null /* rootManifest */, lastRowId, addedRows, "key-1"); @@ -423,6 +425,37 @@ public void testAddSnapshotFromJson() throws IOException { assertEquals(action, expected, MetadataUpdateParser.fromJson(json)); } + @Test + public void testAddSnapshotWithRootManifest() { + String action = MetadataUpdateParser.ADD_SNAPSHOT; + long snapshotId = 2; + int schemaId = 3; + String rootManifest = "s3://bucket/metadata/root-manifest-2.parquet"; + + Snapshot snapshot = + new BaseSnapshot( + 0 /* sequenceNumber */, + snapshotId, + null /* parentId */, + System.currentTimeMillis(), + DataOperations.APPEND, + ImmutableMap.of("files-added", "4") /* summary */, + schemaId, + null /* manifestList */, + rootManifest, + 0L /* firstRowId */, + 10L /* addedRows */, + "key-1" /* keyId */); + String snapshotJson = SnapshotParser.toJson(snapshot, /* pretty */ false); + String json = String.format("{\"action\":\"%s\",\"snapshot\":%s}", action, snapshotJson); + assertThat(json).contains("\"root-manifest\":\"" + rootManifest + "\""); + assertThat(json).doesNotContain("manifest-list"); + + MetadataUpdate expected = new MetadataUpdate.AddSnapshot(snapshot); + assertThat(MetadataUpdateParser.toJson(expected)).isEqualTo(json); + assertEquals(action, expected, MetadataUpdateParser.fromJson(json)); + } + /** RemoveSnapshot * */ @Test public void testRemoveSnapshotFromJson() { @@ -1270,6 +1303,8 @@ private static void assertEqualsAddSnapshot( assertThat(actual.snapshot().snapshotId()).isEqualTo(expected.snapshot().snapshotId()); assertThat(actual.snapshot().manifestListLocation()) .isEqualTo(expected.snapshot().manifestListLocation()); + assertThat(actual.snapshot().rootManifestLocation()) + .isEqualTo(expected.snapshot().rootManifestLocation()); assertThat(actual.snapshot().summary()) .as("Snapshot summary should be equivalent") .containsExactlyEntriesOf(expected.snapshot().summary()); diff --git a/core/src/test/java/org/apache/iceberg/TestRowLineageMetadata.java b/core/src/test/java/org/apache/iceberg/TestRowLineageMetadata.java index 4d861d5a815c..00f9c8e24acf 100644 --- a/core/src/test/java/org/apache/iceberg/TestRowLineageMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestRowLineageMetadata.java @@ -70,40 +70,108 @@ public void cleanup() { public void testSnapshotRowIDValidation() { Snapshot snapshot = new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, null, null); + 0 /* sequenceNumber */, + 1 /* snapshotId */, + null /* parentId */, + 0 /* timestampMillis */, + DataOperations.APPEND, + null /* summary */, + 1 /* schemaId */, + "ml.avro" /* manifestListLocation */, + null /* rootManifest */, + null /* firstRowId */, + null /* addedRows */, + null /* keyId */); assertThat(snapshot.firstRowId()).isNull(); assertThat(snapshot.addedRows()).isNull(); // added-rows will be set to null if first-row-id is null snapshot = - new BaseSnapshot(0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, 10L, null); + new BaseSnapshot( + 0 /* sequenceNumber */, + 1 /* snapshotId */, + null /* parentId */, + 0 /* timestampMillis */, + DataOperations.APPEND, + null /* summary */, + 1 /* schemaId */, + "ml.avro" /* manifestListLocation */, + null /* rootManifest */, + null /* firstRowId */, + 10L /* addedRows */, + null /* keyId */); assertThat(snapshot.firstRowId()).isNull(); assertThat(snapshot.addedRows()).isNull(); // added-rows and first-row-id can be 0 snapshot = - new BaseSnapshot(0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 0L, 0L, null); + new BaseSnapshot( + 0 /* sequenceNumber */, + 1 /* snapshotId */, + null /* parentId */, + 0 /* timestampMillis */, + DataOperations.APPEND, + null /* summary */, + 1 /* schemaId */, + "ml.avro" /* manifestListLocation */, + null /* rootManifest */, + 0L /* firstRowId */, + 0L /* addedRows */, + null /* keyId */); assertThat(snapshot.firstRowId()).isEqualTo(0); assertThat(snapshot.addedRows()).isEqualTo(0); assertThatThrownBy( () -> new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, null, null)) + 0 /* sequenceNumber */, + 1 /* snapshotId */, + null /* parentId */, + 0 /* timestampMillis */, + DataOperations.APPEND, + null /* summary */, + 1 /* schemaId */, + "ml.avro" /* manifestListLocation */, + null /* rootManifest */, + 10L /* firstRowId */, + null /* addedRows */, + null /* keyId */)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Invalid added-rows (required when first-row-id is set): null"); assertThatThrownBy( () -> new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 0L, -1L, null)) + 0 /* sequenceNumber */, + 1 /* snapshotId */, + null /* parentId */, + 0 /* timestampMillis */, + DataOperations.APPEND, + null /* summary */, + 1 /* schemaId */, + "ml.avro" /* manifestListLocation */, + null /* rootManifest */, + 0L /* firstRowId */, + -1L /* addedRows */, + null /* keyId */)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Invalid added-rows (cannot be negative): -1"); assertThatThrownBy( () -> new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", -1L, 1L, null)) + 0 /* sequenceNumber */, + 1 /* snapshotId */, + null /* parentId */, + 0 /* timestampMillis */, + DataOperations.APPEND, + null /* summary */, + 1 /* schemaId */, + "ml.avro" /* manifestListLocation */, + null /* rootManifest */, + -1L /* firstRowId */, + 1L /* addedRows */, + null /* keyId */)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Invalid first-row-id (cannot be negative): -1"); } @@ -116,7 +184,18 @@ public void testSnapshotAddition() { Snapshot addRows = new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "foo", base.nextRowId(), newRows, null); + 0 /* sequenceNumber */, + 1 /* snapshotId */, + null /* parentId */, + 0 /* timestampMillis */, + DataOperations.APPEND, + null /* summary */, + 1 /* schemaId */, + "foo" /* manifestListLocation */, + null /* rootManifest */, + base.nextRowId() /* firstRowId */, + newRows, + null /* keyId */); TableMetadata firstAddition = TableMetadata.buildFrom(base).addSnapshot(addRows).build(); @@ -132,6 +211,7 @@ public void testSnapshotAddition() { null, 1, "foo", + null /* rootManifest */, firstAddition.nextRowId(), newRows, null); @@ -149,7 +229,19 @@ public void testInvalidSnapshotAddition() { TableMetadata base = baseMetadata(); Snapshot invalidLastRow = - new BaseSnapshot(0, 1, null, 0, DataOperations.APPEND, null, 1, "foo", null, newRows, null); + new BaseSnapshot( + 0 /* sequenceNumber */, + 1 /* snapshotId */, + null /* parentId */, + 0 /* timestampMillis */, + DataOperations.APPEND, + null /* summary */, + 1 /* schemaId */, + "foo" /* manifestListLocation */, + null /* rootManifest */, + null /* firstRowId */, + newRows, + null /* keyId */); assertThatThrownBy(() -> TableMetadata.buildFrom(base).addSnapshot(invalidLastRow)) .isInstanceOf(ValidationException.class) @@ -158,12 +250,34 @@ public void testInvalidSnapshotAddition() { // add rows to check TableMetadata validation; Snapshot rejects negative next-row-id Snapshot addRows = new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "foo", base.nextRowId(), newRows, null); + 0 /* sequenceNumber */, + 1 /* snapshotId */, + null /* parentId */, + 0 /* timestampMillis */, + DataOperations.APPEND, + null /* summary */, + 1 /* schemaId */, + "foo" /* manifestListLocation */, + null /* rootManifest */, + base.nextRowId() /* firstRowId */, + newRows, + null /* keyId */); TableMetadata added = TableMetadata.buildFrom(base).addSnapshot(addRows).build(); Snapshot invalidNewRows = new BaseSnapshot( - 1, 2, 1L, 1, DataOperations.APPEND, null, 1, "foo", added.nextRowId() - 1, 10L, null); + 1 /* sequenceNumber */, + 2 /* snapshotId */, + 1L /* parentId */, + 1 /* timestampMillis */, + DataOperations.APPEND, + null /* summary */, + 1 /* schemaId */, + "foo" /* manifestListLocation */, + null /* rootManifest */, + added.nextRowId() - 1 /* firstRowId */, + 10L /* addedRows */, + null /* keyId */); assertThatThrownBy(() -> TableMetadata.buildFrom(added).addSnapshot(invalidNewRows)) .isInstanceOf(ValidationException.class) diff --git a/core/src/test/java/org/apache/iceberg/TestSnapshot.java b/core/src/test/java/org/apache/iceberg/TestSnapshot.java index 398bce3e94d9..9785f333441d 100644 --- a/core/src/test/java/org/apache/iceberg/TestSnapshot.java +++ b/core/src/test/java/org/apache/iceberg/TestSnapshot.java @@ -34,22 +34,88 @@ public class TestSnapshot extends TestBase { public void snapshotValueMethodsIncludeMetadataFields() { Snapshot snapshot = new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 5L, "key-1"); + 0 /* sequenceNumber */, + 1 /* snapshotId */, + null /* parentId */, + 0 /* timestampMillis */, + DataOperations.APPEND, + null /* summary */, + 1 /* schemaId */, + "ml.avro" /* manifestListLocation */, + null /* rootManifest */, + 10L /* firstRowId */, + 5L /* addedRows */, + "key-1" /* keyId */); Snapshot same = new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 5L, "key-1"); + 0 /* sequenceNumber */, + 1 /* snapshotId */, + null /* parentId */, + 0 /* timestampMillis */, + DataOperations.APPEND, + null /* summary */, + 1 /* schemaId */, + "ml.avro" /* manifestListLocation */, + null /* rootManifest */, + 10L /* firstRowId */, + 5L /* addedRows */, + "key-1" /* keyId */); Snapshot differentFirstRowId = new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 11L, 5L, "key-1"); + 0 /* sequenceNumber */, + 1 /* snapshotId */, + null /* parentId */, + 0 /* timestampMillis */, + DataOperations.APPEND, + null /* summary */, + 1 /* schemaId */, + "ml.avro" /* manifestListLocation */, + null /* rootManifest */, + 11L /* firstRowId */, + 5L /* addedRows */, + "key-1" /* keyId */); Snapshot differentAddedRows = new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 6L, "key-1"); + 0 /* sequenceNumber */, + 1 /* snapshotId */, + null /* parentId */, + 0 /* timestampMillis */, + DataOperations.APPEND, + null /* summary */, + 1 /* schemaId */, + "ml.avro" /* manifestListLocation */, + null /* rootManifest */, + 10L /* firstRowId */, + 6L /* addedRows */, + "key-1" /* keyId */); Snapshot differentKeyId = new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 5L, "key-2"); + 0 /* sequenceNumber */, + 1 /* snapshotId */, + null /* parentId */, + 0 /* timestampMillis */, + DataOperations.APPEND, + null /* summary */, + 1 /* schemaId */, + "ml.avro" /* manifestListLocation */, + null /* rootManifest */, + 10L /* firstRowId */, + 5L /* addedRows */, + "key-2" /* keyId */); Snapshot withoutMetadataFields = new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, null, null); + 0 /* sequenceNumber */, + 1 /* snapshotId */, + null /* parentId */, + 0 /* timestampMillis */, + DataOperations.APPEND, + null /* summary */, + 1 /* schemaId */, + "ml.avro" /* manifestListLocation */, + null /* rootManifest */, + null /* firstRowId */, + null /* addedRows */, + null /* keyId */); assertThat(snapshot).isEqualTo(same); assertThat(snapshot.hashCode()).isEqualTo(same.hashCode()); diff --git a/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java b/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java index f16e4feca82b..d7659f3715a3 100644 --- a/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java +++ b/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java @@ -53,6 +53,7 @@ public void testJsonConversion() throws IOException { null, 1, manifestList, + null /* rootManifest */, null, null, keyId); @@ -66,6 +67,8 @@ public void testJsonConversion() throws IOException { assertThat(snapshot.firstRowId()).isNull(); assertThat(snapshot.addedRows()).isNull(); assertThat(snapshot.keyId()).isEqualTo(keyId); + assertThat(snapshot.manifestListLocation()).isEqualTo(expected.manifestListLocation()); + assertThat(snapshot.rootManifestLocation()).isEqualTo(expected.rootManifestLocation()); } @Test @@ -84,6 +87,7 @@ public void testJsonConversionWithoutSchemaId() throws IOException { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -91,6 +95,8 @@ public void testJsonConversionWithoutSchemaId() throws IOException { Snapshot snapshot = SnapshotParser.fromJson(json); assertThat(snapshot.snapshotId()).isEqualTo(expected.snapshotId()); + assertThat(snapshot.manifestListLocation()).isEqualTo(expected.manifestListLocation()); + assertThat(snapshot.rootManifestLocation()).isEqualTo(expected.rootManifestLocation()); assertThat(snapshot.allManifests(ops.io())).isEqualTo(expected.allManifests(ops.io())); assertThat(snapshot.operation()).isNull(); assertThat(snapshot.summary()).isNull(); @@ -116,6 +122,7 @@ public void testJsonConversionWithOperation() throws IOException { ImmutableMap.of("files-added", "4", "files-deleted", "100"), 3, manifestList, + null /* rootManifest */, null, null, null); @@ -130,6 +137,7 @@ public void testJsonConversionWithOperation() throws IOException { assertThat(snapshot.timestampMillis()).isEqualTo(expected.timestampMillis()); assertThat(snapshot.parentId()).isEqualTo(expected.parentId()); assertThat(snapshot.manifestListLocation()).isEqualTo(expected.manifestListLocation()); + assertThat(snapshot.rootManifestLocation()).isEqualTo(expected.rootManifestLocation()); assertThat(snapshot.allManifests(ops.io())).isEqualTo(expected.allManifests(ops.io())); assertThat(snapshot.operation()).isEqualTo(expected.operation()); assertThat(snapshot.summary()).isEqualTo(expected.summary()); @@ -156,6 +164,7 @@ public void testJsonConversionWithRowLineage() throws IOException { null, null, manifestList, + null /* rootManifest */, firstRowId, addedRows, null); @@ -163,6 +172,8 @@ public void testJsonConversionWithRowLineage() throws IOException { Snapshot snapshot = SnapshotParser.fromJson(json); assertThat(snapshot.snapshotId()).isEqualTo(expected.snapshotId()); + assertThat(snapshot.manifestListLocation()).isEqualTo(expected.manifestListLocation()); + assertThat(snapshot.rootManifestLocation()).isEqualTo(expected.rootManifestLocation()); assertThat(snapshot.allManifests(ops.io())).isEqualTo(expected.allManifests(ops.io())); assertThat(snapshot.operation()).isNull(); assertThat(snapshot.summary()).isNull(); @@ -217,6 +228,7 @@ public void testJsonConversionWithV1Manifests() { assertThat(snapshot.timestampMillis()).isEqualTo(expected.timestampMillis()); assertThat(snapshot.parentId()).isEqualTo(expected.parentId()); assertThat(snapshot.manifestListLocation()).isEqualTo(expected.manifestListLocation()); + assertThat(snapshot.rootManifestLocation()).isEqualTo(expected.rootManifestLocation()); assertThat(snapshot.allManifests(ops.io())).isEqualTo(expected.allManifests(ops.io())); assertThat(snapshot.operation()).isEqualTo(expected.operation()); assertThat(snapshot.summary()).isEqualTo(expected.summary()); @@ -320,4 +332,36 @@ public void testJsonConversionEmptySummary() { currentMs); assertThat(SnapshotParser.toJson(snap)).isEqualTo(expected); } + + @Test + public void testRootManifestRoundTrip() { + long timestampMillis = System.currentTimeMillis(); + Snapshot expected = + new BaseSnapshot( + 1 /* sequenceNumber */, + 23 /* snapshotId */, + null /* parentId */, + timestampMillis /* timestampMillis */, + "append" /* operation */, + ImmutableMap.of("operation", "append") /* summary */, + 1 /* schemaId */, + null /* manifestList */, + "s3://bucket/metadata/root-manifest-23.parquet" /* rootManifest */, + 0L /* firstRowId */, + 10L /* addedRows */, + "key-1" /* keyId */); + + String json = SnapshotParser.toJson(expected, false); + assertThat(json) + .contains("\"root-manifest\":\"s3://bucket/metadata/root-manifest-23.parquet\""); + assertThat(json).doesNotContain("manifest-list"); + + Snapshot snapshot = SnapshotParser.fromJson(json); + assertThat(snapshot.manifestListLocation()).isNull(); + assertThat(snapshot.rootManifestLocation()).isEqualTo(expected.rootManifestLocation()); + assertThat(snapshot.keyId()).isEqualTo("key-1"); + assertThat(snapshot.firstRowId()).isEqualTo(0L); + assertThat(snapshot.addedRows()).isEqualTo(10L); + assertThat(SnapshotParser.toJson(snapshot, false)).isEqualTo(json); + } } diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java index 02c23405033a..12028eda2594 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java @@ -120,6 +120,7 @@ public void testJsonConversion() throws Exception { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -138,6 +139,7 @@ public void testJsonConversion() throws Exception { null, 7, manifestList, + null /* rootManifest */, null, null, null); @@ -265,6 +267,7 @@ public void testBackwardCompat() throws Exception { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -283,6 +286,7 @@ public void testBackwardCompat() throws Exception { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -372,6 +376,7 @@ public void testInvalidMainBranch() throws IOException { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -391,6 +396,7 @@ public void testInvalidMainBranch() throws IOException { null, 7, manifestList, + null /* rootManifest */, null, null, null); @@ -452,7 +458,18 @@ public void testMainWithoutCurrent() throws IOException { createManifestListWithManifestFile(snapshotId, null, "file:/tmp/manifest1.avro"); Snapshot snapshot = new BaseSnapshot( - 0, snapshotId, null, snapshotId, null, null, null, manifestList, null, null, null); + 0 /* sequenceNumber */, + snapshotId, + null /* parentId */, + snapshotId /* timestampMillis */, + null /* operation */, + null /* summary */, + null /* schemaId */, + manifestList, + null /* rootManifest */, + null /* firstRowId */, + null /* addedRows */, + null /* keyId */); Schema schema = new Schema(6, Types.NestedField.required(10, "x", Types.StringType.get())); @@ -590,6 +607,7 @@ public void testJsonWithPreviousMetadataLog() throws Exception { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -608,6 +626,7 @@ public void testJsonWithPreviousMetadataLog() throws Exception { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -670,6 +689,7 @@ public void testAddPreviousMetadataRemoveNone() throws IOException { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -688,6 +708,7 @@ public void testAddPreviousMetadataRemoveNone() throws IOException { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -767,6 +788,7 @@ public void testAddPreviousMetadataRemoveOne() throws IOException { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -785,6 +807,7 @@ public void testAddPreviousMetadataRemoveOne() throws IOException { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -877,6 +900,7 @@ public void testAddPreviousMetadataRemoveMultiple() throws IOException { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -895,6 +919,7 @@ public void testAddPreviousMetadataRemoveMultiple() throws IOException { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -2088,6 +2113,7 @@ public void testAddSnapshotWithStaleSequenceNumberIsRetryable() { null, null, "file:/s1.avro", + null /* rootManifest */, null, null, null); @@ -2104,6 +2130,7 @@ public void testAddSnapshotWithStaleSequenceNumberIsRetryable() { null, null, "file:/s2.avro", + null /* rootManifest */, null, null, null); @@ -2133,6 +2160,7 @@ public void testAddSnapshotWithStaleFirstRowIdIsRetryable() { null, null, "file:/s1.avro", + null /* rootManifest */, 0L, 5L, null); @@ -2142,7 +2170,18 @@ public void testAddSnapshotWithStaleFirstRowIdIsRetryable() { // A snapshot with firstRowId=0 is stale (0 < nextRowId=5) Snapshot staleSnapshot = new BaseSnapshot( - 2, 2L, 1L, System.currentTimeMillis(), null, null, null, "file:/s2.avro", 0L, 1L, null); + 2 /* sequenceNumber */, + 2L /* snapshotId */, + 1L /* parentId */, + System.currentTimeMillis(), + null /* operation */, + null /* summary */, + null /* schemaId */, + "file:/s2.avro" /* manifestListLocation */, + null /* rootManifest */, + 0L /* firstRowId */, + 1L /* addedRows */, + null /* keyId */); assertThatThrownBy(() -> TableMetadata.buildFrom(withS1).addSnapshot(staleSnapshot)) .isInstanceOf(RetryableValidationException.class) @@ -2165,6 +2204,7 @@ public void testSetRefRejectsTagForMainBranch() { null, null, "file:/s1.avro", + null /* rootManifest */, null, null, null); @@ -2198,11 +2238,35 @@ public void addSnapshotRejectsNonMonotonicTimestampAfterV4(int formatVersion) { ImmutableMap.of(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion))); Snapshot parent = - new BaseSnapshot(1, 1L, null, 1_000L, null, null, null, "file:/s1.avro", 0L, 0L, null); + new BaseSnapshot( + 1 /* sequenceNumber */, + 1L /* snapshotId */, + null /* parentId */, + 1_000L /* timestampMillis */, + null /* operation */, + null /* summary */, + null /* schemaId */, + "file:/s1.avro" /* manifestListLocation */, + null /* rootManifest */, + 0L /* firstRowId */, + 0L /* addedRows */, + null /* keyId */); TableMetadata withParent = TableMetadata.buildFrom(base).addSnapshot(parent).build(); Snapshot child = - new BaseSnapshot(2, 2L, 1L, 1_000L, null, null, null, "file:/s2.avro", 0L, 0L, null); + new BaseSnapshot( + 2 /* sequenceNumber */, + 2L /* snapshotId */, + 1L /* parentId */, + 1_000L /* timestampMillis */, + null /* operation */, + null /* summary */, + null /* schemaId */, + "file:/s2.avro" /* manifestListLocation */, + null /* rootManifest */, + 0L /* firstRowId */, + 0L /* addedRows */, + null /* keyId */); assertThatThrownBy(() -> TableMetadata.buildFrom(withParent).addSnapshot(child)) .isInstanceOf(ValidationException.class) @@ -2235,6 +2299,7 @@ public void addSnapshotAllowsNonMonotonicTimestampBeforeV4(int formatVersion) { null, null, "file:/s1.avro", + null /* rootManifest */, parentFirstRowId, parentAddedRows, null); @@ -2255,6 +2320,7 @@ public void addSnapshotAllowsNonMonotonicTimestampBeforeV4(int formatVersion) { null, null, "file:/s2.avro", + null /* rootManifest */, childFirstRowId, childAddedRows, null); @@ -2288,6 +2354,7 @@ public void snapshotTimestampMayBeAheadOfTableMetadataLastUpdatedMillis(int form null, null, "file:/s1.avro", + null /* rootManifest */, firstRowId, addedRows, null); From 197d674de66ec5f1d13f8dee15bfed4d000611d1 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Thu, 12 Nov 2026 15:36:36 -0800 Subject: [PATCH 2/4] API, Core: Require a snapshot manifest list or root manifest A snapshot must have exactly one of manifest-list or root-manifest. Reject reading manifests from a root-manifest snapshot until that path is implemented. Generated-by: Cursor Co-authored-by: Cursor --- .../java/org/apache/iceberg/Snapshot.java | 12 +- .../java/org/apache/iceberg/BaseSnapshot.java | 15 +- .../org/apache/iceberg/SnapshotParser.java | 1 - .../iceberg/TestMetadataUpdateParser.java | 3 +- .../iceberg/TestRewriteTablePathUtil.java | 34 ++++ .../iceberg/TestRowLineageMetadata.java | 162 ++++++------------ .../java/org/apache/iceberg/TestSnapshot.java | 149 +++++++++------- .../org/apache/iceberg/TestSnapshotJson.java | 13 +- .../org/apache/iceberg/TestTableMetadata.java | 69 +++----- 9 files changed, 209 insertions(+), 249 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/Snapshot.java b/api/src/main/java/org/apache/iceberg/Snapshot.java index 28efa6255938..ca9a95c4f5da 100644 --- a/api/src/main/java/org/apache/iceberg/Snapshot.java +++ b/api/src/main/java/org/apache/iceberg/Snapshot.java @@ -166,19 +166,13 @@ default Iterable removedDeleteFiles(FileIO io) { /** * Return the location of this snapshot's manifest list, or null if it is not separate. * - *

Only set for tables with format version 1 through 3. Snapshots in v4 tables use {@link - * #rootManifestLocation()} instead. - * - * @return the location of the manifest list for this snapshot + * @return the location of the manifest list for this Snapshot */ String manifestListLocation(); /** - * Return the location of this snapshot's root manifest, or null if this snapshot has no root - * manifest. - * - *

v4 snapshots store this in {@code root-manifest} instead of a manifest list. At most one of - * this and {@link #manifestListLocation()} is set. + * Return the location of this snapshot's root manifest, or null if this snapshot does not have + * one. * * @return the location of the root manifest for this snapshot */ diff --git a/core/src/main/java/org/apache/iceberg/BaseSnapshot.java b/core/src/main/java/org/apache/iceberg/BaseSnapshot.java index aa079fc1521b..38aa50d7652b 100644 --- a/core/src/main/java/org/apache/iceberg/BaseSnapshot.java +++ b/core/src/main/java/org/apache/iceberg/BaseSnapshot.java @@ -74,6 +74,10 @@ class BaseSnapshot implements Snapshot { manifestList == null || rootManifest == null, "Snapshot %s cannot have both a manifest list and a root manifest", snapshotId); + Preconditions.checkArgument( + manifestList != null || rootManifest != null, + "Snapshot %s must have a manifest list or a root manifest", + snapshotId); Preconditions.checkArgument( firstRowId == null || firstRowId >= 0, "Invalid first-row-id (cannot be negative): %s", @@ -175,6 +179,11 @@ public String keyId() { } private void cacheManifests(FileIO fileIO) { + Preconditions.checkState( + rootManifestLocation == null, + "Cannot read manifests from snapshot %s: root manifests are not supported", + snapshotId); + if (fileIO == null) { throw new IllegalArgumentException("Cannot cache changes: FileIO is null"); } @@ -189,11 +198,7 @@ private void cacheManifests(FileIO fileIO) { } if (allManifests == null) { - // if manifests isn't set, then the manifest list must be read - Preconditions.checkArgument( - manifestListLocation != null, - "Cannot read manifests from snapshot %s: no manifest list", - snapshotId); + // if manifests isn't set, then the snapshotFile is set and should be read to get the list this.allManifests = ManifestLists.read( ManifestLists.newInputFile( diff --git a/core/src/main/java/org/apache/iceberg/SnapshotParser.java b/core/src/main/java/org/apache/iceberg/SnapshotParser.java index fd896f1df5a5..29e56d9d7009 100644 --- a/core/src/main/java/org/apache/iceberg/SnapshotParser.java +++ b/core/src/main/java/org/apache/iceberg/SnapshotParser.java @@ -184,7 +184,6 @@ static Snapshot fromJson(JsonNode node) { String rootManifest = JsonUtil.getStringOrNull(ROOT_MANIFEST, node); if (manifestList != null || rootManifest != null) { - // manifest-list is used by v1-v3 snapshots, root-manifest by v4 snapshots return new BaseSnapshot( sequenceNumber, snapshotId, diff --git a/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java b/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java index 9665c53adb1f..7386b5734d97 100644 --- a/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java +++ b/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java @@ -429,7 +429,6 @@ public void testAddSnapshotFromJson() throws IOException { public void testAddSnapshotWithRootManifest() { String action = MetadataUpdateParser.ADD_SNAPSHOT; long snapshotId = 2; - int schemaId = 3; String rootManifest = "s3://bucket/metadata/root-manifest-2.parquet"; Snapshot snapshot = @@ -440,7 +439,7 @@ public void testAddSnapshotWithRootManifest() { System.currentTimeMillis(), DataOperations.APPEND, ImmutableMap.of("files-added", "4") /* summary */, - schemaId, + 3 /* schemaId */, null /* manifestList */, rootManifest, 0L /* firstRowId */, diff --git a/core/src/test/java/org/apache/iceberg/TestRewriteTablePathUtil.java b/core/src/test/java/org/apache/iceberg/TestRewriteTablePathUtil.java index 64429dc3c12b..38ecf3509455 100644 --- a/core/src/test/java/org/apache/iceberg/TestRewriteTablePathUtil.java +++ b/core/src/test/java/org/apache/iceberg/TestRewriteTablePathUtil.java @@ -303,6 +303,40 @@ public void testReplacePathsWithTrailingSeparatorInPrefix() { assertThat(replaced.location()).isEqualTo("s3://bucket/restored/table"); } + @Test + void replacePathsRewritesRootManifestLocation() { + String rootManifestPath = "/metadata/root-manifest.parquet"; + String sourcePrefix = "s3://bucket/warehouse/table"; + String targetPrefix = "s3://bucket/restored/table"; + String sourceRootManifest = sourcePrefix + rootManifestPath; + String targetRootManifest = targetPrefix + rootManifestPath; + TableMetadata metadata = + TableMetadata.newTableMetadata( + SCHEMA, PartitionSpec.unpartitioned(), sourcePrefix, ImmutableMap.of()); + Snapshot snapshot = + new BaseSnapshot( + 1 /* sequenceNumber */, + 1L /* snapshotId */, + null /* parentId */, + System.currentTimeMillis(), + DataOperations.APPEND, + null /* summary */, + metadata.currentSchemaId(), + null /* manifestList */, + sourceRootManifest, + null /* firstRowId */, + null /* addedRows */, + null /* keyId */); + TableMetadata withRootManifest = + TableMetadata.buildFrom(metadata).addSnapshot(snapshot).build(); + + TableMetadata replaced = + RewriteTablePathUtil.replacePaths(withRootManifest, sourcePrefix, targetPrefix); + + assertThat(replaced.snapshot(snapshot.snapshotId()).rootManifestLocation()) + .isEqualTo(targetRootManifest); + } + @Test public void testNewPathTableRename() { // Rename /tableX to /table (target is substring of source name) diff --git a/core/src/test/java/org/apache/iceberg/TestRowLineageMetadata.java b/core/src/test/java/org/apache/iceberg/TestRowLineageMetadata.java index 00f9c8e24acf..7b490b08560f 100644 --- a/core/src/test/java/org/apache/iceberg/TestRowLineageMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestRowLineageMetadata.java @@ -70,108 +70,53 @@ public void cleanup() { public void testSnapshotRowIDValidation() { Snapshot snapshot = new BaseSnapshot( - 0 /* sequenceNumber */, - 1 /* snapshotId */, - null /* parentId */, - 0 /* timestampMillis */, - DataOperations.APPEND, - null /* summary */, - 1 /* schemaId */, - "ml.avro" /* manifestListLocation */, - null /* rootManifest */, - null /* firstRowId */, - null /* addedRows */, - null /* keyId */); + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, null, null, null); assertThat(snapshot.firstRowId()).isNull(); assertThat(snapshot.addedRows()).isNull(); // added-rows will be set to null if first-row-id is null snapshot = new BaseSnapshot( - 0 /* sequenceNumber */, - 1 /* snapshotId */, - null /* parentId */, - 0 /* timestampMillis */, - DataOperations.APPEND, - null /* summary */, - 1 /* schemaId */, - "ml.avro" /* manifestListLocation */, - null /* rootManifest */, - null /* firstRowId */, - 10L /* addedRows */, - null /* keyId */); + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, null, 10L, null); assertThat(snapshot.firstRowId()).isNull(); assertThat(snapshot.addedRows()).isNull(); // added-rows and first-row-id can be 0 snapshot = new BaseSnapshot( - 0 /* sequenceNumber */, - 1 /* snapshotId */, - null /* parentId */, - 0 /* timestampMillis */, - DataOperations.APPEND, - null /* summary */, - 1 /* schemaId */, - "ml.avro" /* manifestListLocation */, - null /* rootManifest */, - 0L /* firstRowId */, - 0L /* addedRows */, - null /* keyId */); + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, 0L, 0L, null); assertThat(snapshot.firstRowId()).isEqualTo(0); assertThat(snapshot.addedRows()).isEqualTo(0); assertThatThrownBy( () -> new BaseSnapshot( - 0 /* sequenceNumber */, - 1 /* snapshotId */, - null /* parentId */, - 0 /* timestampMillis */, + 0, + 1, + null, + 0, DataOperations.APPEND, - null /* summary */, - 1 /* schemaId */, - "ml.avro" /* manifestListLocation */, - null /* rootManifest */, - 10L /* firstRowId */, - null /* addedRows */, - null /* keyId */)) + null, + 1, + "ml.avro", + null, + 10L, + null, + null)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Invalid added-rows (required when first-row-id is set): null"); assertThatThrownBy( () -> new BaseSnapshot( - 0 /* sequenceNumber */, - 1 /* snapshotId */, - null /* parentId */, - 0 /* timestampMillis */, - DataOperations.APPEND, - null /* summary */, - 1 /* schemaId */, - "ml.avro" /* manifestListLocation */, - null /* rootManifest */, - 0L /* firstRowId */, - -1L /* addedRows */, - null /* keyId */)) + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, 0L, -1L, null)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Invalid added-rows (cannot be negative): -1"); assertThatThrownBy( () -> new BaseSnapshot( - 0 /* sequenceNumber */, - 1 /* snapshotId */, - null /* parentId */, - 0 /* timestampMillis */, - DataOperations.APPEND, - null /* summary */, - 1 /* schemaId */, - "ml.avro" /* manifestListLocation */, - null /* rootManifest */, - -1L /* firstRowId */, - 1L /* addedRows */, - null /* keyId */)) + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, -1L, 1L, null)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Invalid first-row-id (cannot be negative): -1"); } @@ -184,18 +129,18 @@ public void testSnapshotAddition() { Snapshot addRows = new BaseSnapshot( - 0 /* sequenceNumber */, - 1 /* snapshotId */, - null /* parentId */, - 0 /* timestampMillis */, + 0, + 1, + null, + 0, DataOperations.APPEND, - null /* summary */, - 1 /* schemaId */, - "foo" /* manifestListLocation */, - null /* rootManifest */, - base.nextRowId() /* firstRowId */, + null, + 1, + "foo", + null, + base.nextRowId(), newRows, - null /* keyId */); + null); TableMetadata firstAddition = TableMetadata.buildFrom(base).addSnapshot(addRows).build(); @@ -230,18 +175,7 @@ public void testInvalidSnapshotAddition() { Snapshot invalidLastRow = new BaseSnapshot( - 0 /* sequenceNumber */, - 1 /* snapshotId */, - null /* parentId */, - 0 /* timestampMillis */, - DataOperations.APPEND, - null /* summary */, - 1 /* schemaId */, - "foo" /* manifestListLocation */, - null /* rootManifest */, - null /* firstRowId */, - newRows, - null /* keyId */); + 0, 1, null, 0, DataOperations.APPEND, null, 1, "foo", null, null, newRows, null); assertThatThrownBy(() -> TableMetadata.buildFrom(base).addSnapshot(invalidLastRow)) .isInstanceOf(ValidationException.class) @@ -250,34 +184,34 @@ public void testInvalidSnapshotAddition() { // add rows to check TableMetadata validation; Snapshot rejects negative next-row-id Snapshot addRows = new BaseSnapshot( - 0 /* sequenceNumber */, - 1 /* snapshotId */, - null /* parentId */, - 0 /* timestampMillis */, + 0, + 1, + null, + 0, DataOperations.APPEND, - null /* summary */, - 1 /* schemaId */, - "foo" /* manifestListLocation */, - null /* rootManifest */, - base.nextRowId() /* firstRowId */, + null, + 1, + "foo", + null, + base.nextRowId(), newRows, - null /* keyId */); + null); TableMetadata added = TableMetadata.buildFrom(base).addSnapshot(addRows).build(); Snapshot invalidNewRows = new BaseSnapshot( - 1 /* sequenceNumber */, - 2 /* snapshotId */, - 1L /* parentId */, - 1 /* timestampMillis */, + 1, + 2, + 1L, + 1, DataOperations.APPEND, - null /* summary */, - 1 /* schemaId */, - "foo" /* manifestListLocation */, - null /* rootManifest */, - added.nextRowId() - 1 /* firstRowId */, - 10L /* addedRows */, - null /* keyId */); + null, + 1, + "foo", + null, + added.nextRowId() - 1, + 10L, + null); assertThatThrownBy(() -> TableMetadata.buildFrom(added).addSnapshot(invalidNewRows)) .isInstanceOf(ValidationException.class) diff --git a/core/src/test/java/org/apache/iceberg/TestSnapshot.java b/core/src/test/java/org/apache/iceberg/TestSnapshot.java index 9785f333441d..afc4a7165e82 100644 --- a/core/src/test/java/org/apache/iceberg/TestSnapshot.java +++ b/core/src/test/java/org/apache/iceberg/TestSnapshot.java @@ -19,6 +19,7 @@ package org.apache.iceberg; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assumptions.assumeThat; import org.apache.iceberg.expressions.Expressions; @@ -34,61 +35,35 @@ public class TestSnapshot extends TestBase { public void snapshotValueMethodsIncludeMetadataFields() { Snapshot snapshot = new BaseSnapshot( - 0 /* sequenceNumber */, - 1 /* snapshotId */, - null /* parentId */, - 0 /* timestampMillis */, - DataOperations.APPEND, - null /* summary */, - 1 /* schemaId */, - "ml.avro" /* manifestListLocation */, - null /* rootManifest */, - 10L /* firstRowId */, - 5L /* addedRows */, - "key-1" /* keyId */); + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, 10L, 5L, "key-1"); Snapshot same = new BaseSnapshot( - 0 /* sequenceNumber */, - 1 /* snapshotId */, - null /* parentId */, - 0 /* timestampMillis */, - DataOperations.APPEND, - null /* summary */, - 1 /* schemaId */, - "ml.avro" /* manifestListLocation */, - null /* rootManifest */, - 10L /* firstRowId */, - 5L /* addedRows */, - "key-1" /* keyId */); + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, 10L, 5L, "key-1"); Snapshot differentFirstRowId = new BaseSnapshot( - 0 /* sequenceNumber */, - 1 /* snapshotId */, - null /* parentId */, - 0 /* timestampMillis */, - DataOperations.APPEND, - null /* summary */, - 1 /* schemaId */, - "ml.avro" /* manifestListLocation */, - null /* rootManifest */, - 11L /* firstRowId */, - 5L /* addedRows */, - "key-1" /* keyId */); + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, 11L, 5L, "key-1"); Snapshot differentAddedRows = new BaseSnapshot( - 0 /* sequenceNumber */, - 1 /* snapshotId */, - null /* parentId */, - 0 /* timestampMillis */, - DataOperations.APPEND, - null /* summary */, - 1 /* schemaId */, - "ml.avro" /* manifestListLocation */, - null /* rootManifest */, - 10L /* firstRowId */, - 6L /* addedRows */, - "key-1" /* keyId */); + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, 10L, 6L, "key-1"); Snapshot differentKeyId = + new BaseSnapshot( + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, 10L, 5L, "key-2"); + Snapshot withoutMetadataFields = + new BaseSnapshot( + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, null, null, null); + + assertThat(snapshot).isEqualTo(same); + assertThat(snapshot.hashCode()).isEqualTo(same.hashCode()); + assertThat(snapshot).isNotEqualTo(differentFirstRowId); + assertThat(snapshot).isNotEqualTo(differentAddedRows); + assertThat(snapshot).isNotEqualTo(differentKeyId); + assertThat(snapshot).isNotEqualTo(withoutMetadataFields); + assertThat(snapshot.toString()).contains("first-row-id=10", "added-rows=5", "key-id=key-1"); + } + + @TestTemplate + void snapshotWithRootManifestExposesLocation() { + Snapshot snapshot = new BaseSnapshot( 0 /* sequenceNumber */, 1 /* snapshotId */, @@ -97,12 +72,62 @@ public void snapshotValueMethodsIncludeMetadataFields() { DataOperations.APPEND, null /* summary */, 1 /* schemaId */, - "ml.avro" /* manifestListLocation */, - null /* rootManifest */, + null /* manifestList */, + "s3://bucket/metadata/root-manifest.parquet" /* rootManifest */, 10L /* firstRowId */, 5L /* addedRows */, - "key-2" /* keyId */); - Snapshot withoutMetadataFields = + "key-1" /* keyId */); + + assertThat(snapshot.rootManifestLocation()) + .isEqualTo("s3://bucket/metadata/root-manifest.parquet"); + assertThat(snapshot.manifestListLocation()).isNull(); + } + + @TestTemplate + void rejectsBothManifestListAndRootManifest() { + assertThatThrownBy( + () -> + new BaseSnapshot( + 0 /* sequenceNumber */, + 1 /* snapshotId */, + null /* parentId */, + 0 /* timestampMillis */, + DataOperations.APPEND, + null /* summary */, + 1 /* schemaId */, + "ml.avro" /* manifestList */, + "root-manifest.parquet" /* rootManifest */, + 10L /* firstRowId */, + 5L /* addedRows */, + "key-1" /* keyId */)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("cannot have both a manifest list and a root manifest"); + } + + @TestTemplate + void rejectsNeitherManifestListNorRootManifest() { + assertThatThrownBy( + () -> + new BaseSnapshot( + 0 /* sequenceNumber */, + 1 /* snapshotId */, + null /* parentId */, + 0 /* timestampMillis */, + DataOperations.APPEND, + null /* summary */, + 1 /* schemaId */, + null /* manifestList */, + null /* rootManifest */, + 10L /* firstRowId */, + 5L /* addedRows */, + "key-1" /* keyId */)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("must have a manifest list or a root manifest"); + } + + @TestTemplate + void allManifestsRejectsRootManifest() { + Snapshot snapshot = new BaseSnapshot( 0 /* sequenceNumber */, 1 /* snapshotId */, @@ -111,19 +136,15 @@ public void snapshotValueMethodsIncludeMetadataFields() { DataOperations.APPEND, null /* summary */, 1 /* schemaId */, - "ml.avro" /* manifestListLocation */, - null /* rootManifest */, - null /* firstRowId */, - null /* addedRows */, - null /* keyId */); + null /* manifestList */, + "s3://bucket/metadata/root-manifest.parquet" /* rootManifest */, + 10L /* firstRowId */, + 5L /* addedRows */, + "key-1" /* keyId */); - assertThat(snapshot).isEqualTo(same); - assertThat(snapshot.hashCode()).isEqualTo(same.hashCode()); - assertThat(snapshot).isNotEqualTo(differentFirstRowId); - assertThat(snapshot).isNotEqualTo(differentAddedRows); - assertThat(snapshot).isNotEqualTo(differentKeyId); - assertThat(snapshot).isNotEqualTo(withoutMetadataFields); - assertThat(snapshot.toString()).contains("first-row-id=10", "added-rows=5", "key-id=key-1"); + assertThatThrownBy(() -> snapshot.allManifests(table.io())) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("root manifests are not supported"); } @TestTemplate diff --git a/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java b/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java index d7659f3715a3..d6020ce32a35 100644 --- a/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java +++ b/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java @@ -67,8 +67,7 @@ public void testJsonConversion() throws IOException { assertThat(snapshot.firstRowId()).isNull(); assertThat(snapshot.addedRows()).isNull(); assertThat(snapshot.keyId()).isEqualTo(keyId); - assertThat(snapshot.manifestListLocation()).isEqualTo(expected.manifestListLocation()); - assertThat(snapshot.rootManifestLocation()).isEqualTo(expected.rootManifestLocation()); + assertThat(snapshot.rootManifestLocation()).isNull(); } @Test @@ -95,8 +94,7 @@ public void testJsonConversionWithoutSchemaId() throws IOException { Snapshot snapshot = SnapshotParser.fromJson(json); assertThat(snapshot.snapshotId()).isEqualTo(expected.snapshotId()); - assertThat(snapshot.manifestListLocation()).isEqualTo(expected.manifestListLocation()); - assertThat(snapshot.rootManifestLocation()).isEqualTo(expected.rootManifestLocation()); + assertThat(snapshot.rootManifestLocation()).isNull(); assertThat(snapshot.allManifests(ops.io())).isEqualTo(expected.allManifests(ops.io())); assertThat(snapshot.operation()).isNull(); assertThat(snapshot.summary()).isNull(); @@ -137,7 +135,7 @@ public void testJsonConversionWithOperation() throws IOException { assertThat(snapshot.timestampMillis()).isEqualTo(expected.timestampMillis()); assertThat(snapshot.parentId()).isEqualTo(expected.parentId()); assertThat(snapshot.manifestListLocation()).isEqualTo(expected.manifestListLocation()); - assertThat(snapshot.rootManifestLocation()).isEqualTo(expected.rootManifestLocation()); + assertThat(snapshot.rootManifestLocation()).isNull(); assertThat(snapshot.allManifests(ops.io())).isEqualTo(expected.allManifests(ops.io())); assertThat(snapshot.operation()).isEqualTo(expected.operation()); assertThat(snapshot.summary()).isEqualTo(expected.summary()); @@ -172,8 +170,7 @@ public void testJsonConversionWithRowLineage() throws IOException { Snapshot snapshot = SnapshotParser.fromJson(json); assertThat(snapshot.snapshotId()).isEqualTo(expected.snapshotId()); - assertThat(snapshot.manifestListLocation()).isEqualTo(expected.manifestListLocation()); - assertThat(snapshot.rootManifestLocation()).isEqualTo(expected.rootManifestLocation()); + assertThat(snapshot.rootManifestLocation()).isNull(); assertThat(snapshot.allManifests(ops.io())).isEqualTo(expected.allManifests(ops.io())); assertThat(snapshot.operation()).isNull(); assertThat(snapshot.summary()).isNull(); @@ -228,7 +225,7 @@ public void testJsonConversionWithV1Manifests() { assertThat(snapshot.timestampMillis()).isEqualTo(expected.timestampMillis()); assertThat(snapshot.parentId()).isEqualTo(expected.parentId()); assertThat(snapshot.manifestListLocation()).isEqualTo(expected.manifestListLocation()); - assertThat(snapshot.rootManifestLocation()).isEqualTo(expected.rootManifestLocation()); + assertThat(snapshot.rootManifestLocation()).isNull(); assertThat(snapshot.allManifests(ops.io())).isEqualTo(expected.allManifests(ops.io())); assertThat(snapshot.operation()).isEqualTo(expected.operation()); assertThat(snapshot.summary()).isEqualTo(expected.summary()); diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java index 12028eda2594..bdf94c87edd0 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java @@ -458,18 +458,18 @@ public void testMainWithoutCurrent() throws IOException { createManifestListWithManifestFile(snapshotId, null, "file:/tmp/manifest1.avro"); Snapshot snapshot = new BaseSnapshot( - 0 /* sequenceNumber */, + 0, + snapshotId, + null, snapshotId, - null /* parentId */, - snapshotId /* timestampMillis */, - null /* operation */, - null /* summary */, - null /* schemaId */, + null, + null, + null, manifestList, - null /* rootManifest */, - null /* firstRowId */, - null /* addedRows */, - null /* keyId */); + null, + null, + null, + null); Schema schema = new Schema(6, Types.NestedField.required(10, "x", Types.StringType.get())); @@ -2170,18 +2170,18 @@ public void testAddSnapshotWithStaleFirstRowIdIsRetryable() { // A snapshot with firstRowId=0 is stale (0 < nextRowId=5) Snapshot staleSnapshot = new BaseSnapshot( - 2 /* sequenceNumber */, - 2L /* snapshotId */, - 1L /* parentId */, + 2, + 2L, + 1L, System.currentTimeMillis(), - null /* operation */, - null /* summary */, - null /* schemaId */, - "file:/s2.avro" /* manifestListLocation */, - null /* rootManifest */, - 0L /* firstRowId */, - 1L /* addedRows */, - null /* keyId */); + null, + null, + null, + "file:/s2.avro", + null, + 0L, + 1L, + null); assertThatThrownBy(() -> TableMetadata.buildFrom(withS1).addSnapshot(staleSnapshot)) .isInstanceOf(RetryableValidationException.class) @@ -2239,34 +2239,11 @@ public void addSnapshotRejectsNonMonotonicTimestampAfterV4(int formatVersion) { Snapshot parent = new BaseSnapshot( - 1 /* sequenceNumber */, - 1L /* snapshotId */, - null /* parentId */, - 1_000L /* timestampMillis */, - null /* operation */, - null /* summary */, - null /* schemaId */, - "file:/s1.avro" /* manifestListLocation */, - null /* rootManifest */, - 0L /* firstRowId */, - 0L /* addedRows */, - null /* keyId */); + 1, 1L, null, 1_000L, null, null, null, "file:/s1.avro", null, 0L, 0L, null); TableMetadata withParent = TableMetadata.buildFrom(base).addSnapshot(parent).build(); Snapshot child = - new BaseSnapshot( - 2 /* sequenceNumber */, - 2L /* snapshotId */, - 1L /* parentId */, - 1_000L /* timestampMillis */, - null /* operation */, - null /* summary */, - null /* schemaId */, - "file:/s2.avro" /* manifestListLocation */, - null /* rootManifest */, - 0L /* firstRowId */, - 0L /* addedRows */, - null /* keyId */); + new BaseSnapshot(2, 2L, 1L, 1_000L, null, null, null, "file:/s2.avro", null, 0L, 0L, null); assertThatThrownBy(() -> TableMetadata.buildFrom(withParent).addSnapshot(child)) .isInstanceOf(ValidationException.class) From 0d47d5c67596b597751bac2ce29abf9e87050d8f Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Thu, 12 Nov 2026 15:47:01 -0800 Subject: [PATCH 3/4] Core: Clarify manifest list and root manifest location docs The manifestListLocation() Javadoc should say when the value is null without referring to whether the list is separate. Also tidy the allManifests comment and the SnapshotProducer root manifest argument so they describe the current code. Generated-by: Cursor Co-authored-by: Cursor --- api/src/main/java/org/apache/iceberg/Snapshot.java | 6 +++--- core/src/main/java/org/apache/iceberg/BaseSnapshot.java | 2 +- core/src/main/java/org/apache/iceberg/SnapshotProducer.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/Snapshot.java b/api/src/main/java/org/apache/iceberg/Snapshot.java index ca9a95c4f5da..8b276c746139 100644 --- a/api/src/main/java/org/apache/iceberg/Snapshot.java +++ b/api/src/main/java/org/apache/iceberg/Snapshot.java @@ -164,15 +164,15 @@ default Iterable removedDeleteFiles(FileIO io) { } /** - * Return the location of this snapshot's manifest list, or null if it is not separate. + * Return the location of this snapshot's manifest list, or null if this snapshot has no manifest + * list. * * @return the location of the manifest list for this Snapshot */ String manifestListLocation(); /** - * Return the location of this snapshot's root manifest, or null if this snapshot does not have - * one. + * Return the location of this snapshot's root manifest, or null if it has no root manifest. * * @return the location of the root manifest for this snapshot */ diff --git a/core/src/main/java/org/apache/iceberg/BaseSnapshot.java b/core/src/main/java/org/apache/iceberg/BaseSnapshot.java index 38aa50d7652b..bd311d1ffa62 100644 --- a/core/src/main/java/org/apache/iceberg/BaseSnapshot.java +++ b/core/src/main/java/org/apache/iceberg/BaseSnapshot.java @@ -198,7 +198,7 @@ private void cacheManifests(FileIO fileIO) { } if (allManifests == null) { - // if manifests isn't set, then the snapshotFile is set and should be read to get the list + // if manifests isn't set, then the manifest list must be read this.allManifests = ManifestLists.read( ManifestLists.newInputFile( diff --git a/core/src/main/java/org/apache/iceberg/SnapshotProducer.java b/core/src/main/java/org/apache/iceberg/SnapshotProducer.java index 48f5351e44ee..74c89b603c0b 100644 --- a/core/src/main/java/org/apache/iceberg/SnapshotProducer.java +++ b/core/src/main/java/org/apache/iceberg/SnapshotProducer.java @@ -372,7 +372,7 @@ public Snapshot apply() { summary(base), base.currentSchemaId(), manifestList.location(), - null /* rootManifest */, + null, // rootManifest nextRowId, assignedRows, manifestListEncryptionKeys != null ? manifestListEncryptionKeys.fileKey().keyId() : null); From f55844ad942ff6b02de7e08085221b0544a7f78f Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Thu, 12 Nov 2026 15:56:34 -0800 Subject: [PATCH 4/4] Core: Use short root manifest names in snapshot tests Test fixtures used full s3 URIs for root manifests while the rest of the snapshot tests use relative names. Keep the URIs only where the test rewrites locations between prefixes. Generated-by: Cursor Co-authored-by: Cursor --- .../java/org/apache/iceberg/TestMetadataUpdateParser.java | 2 +- core/src/test/java/org/apache/iceberg/TestSnapshot.java | 7 +++---- .../src/test/java/org/apache/iceberg/TestSnapshotJson.java | 5 ++--- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java b/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java index 7386b5734d97..4d36e985c5b9 100644 --- a/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java +++ b/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java @@ -429,7 +429,7 @@ public void testAddSnapshotFromJson() throws IOException { public void testAddSnapshotWithRootManifest() { String action = MetadataUpdateParser.ADD_SNAPSHOT; long snapshotId = 2; - String rootManifest = "s3://bucket/metadata/root-manifest-2.parquet"; + String rootManifest = "root-manifest-2.parquet"; Snapshot snapshot = new BaseSnapshot( diff --git a/core/src/test/java/org/apache/iceberg/TestSnapshot.java b/core/src/test/java/org/apache/iceberg/TestSnapshot.java index afc4a7165e82..0f5a3f431766 100644 --- a/core/src/test/java/org/apache/iceberg/TestSnapshot.java +++ b/core/src/test/java/org/apache/iceberg/TestSnapshot.java @@ -73,13 +73,12 @@ void snapshotWithRootManifestExposesLocation() { null /* summary */, 1 /* schemaId */, null /* manifestList */, - "s3://bucket/metadata/root-manifest.parquet" /* rootManifest */, + "root-manifest.parquet" /* rootManifest */, 10L /* firstRowId */, 5L /* addedRows */, "key-1" /* keyId */); - assertThat(snapshot.rootManifestLocation()) - .isEqualTo("s3://bucket/metadata/root-manifest.parquet"); + assertThat(snapshot.rootManifestLocation()).isEqualTo("root-manifest.parquet"); assertThat(snapshot.manifestListLocation()).isNull(); } @@ -137,7 +136,7 @@ void allManifestsRejectsRootManifest() { null /* summary */, 1 /* schemaId */, null /* manifestList */, - "s3://bucket/metadata/root-manifest.parquet" /* rootManifest */, + "root-manifest.parquet" /* rootManifest */, 10L /* firstRowId */, 5L /* addedRows */, "key-1" /* keyId */); diff --git a/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java b/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java index d6020ce32a35..f715642df1c3 100644 --- a/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java +++ b/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java @@ -343,14 +343,13 @@ public void testRootManifestRoundTrip() { ImmutableMap.of("operation", "append") /* summary */, 1 /* schemaId */, null /* manifestList */, - "s3://bucket/metadata/root-manifest-23.parquet" /* rootManifest */, + "root-manifest-23.parquet" /* rootManifest */, 0L /* firstRowId */, 10L /* addedRows */, "key-1" /* keyId */); String json = SnapshotParser.toJson(expected, false); - assertThat(json) - .contains("\"root-manifest\":\"s3://bucket/metadata/root-manifest-23.parquet\""); + assertThat(json).contains("\"root-manifest\":\"root-manifest-23.parquet\""); assertThat(json).doesNotContain("manifest-list"); Snapshot snapshot = SnapshotParser.fromJson(json);