diff --git a/api/src/main/java/org/apache/iceberg/Snapshot.java b/api/src/main/java/org/apache/iceberg/Snapshot.java index 8a74dca6d053..8b276c746139 100644 --- a/api/src/main/java/org/apache/iceberg/Snapshot.java +++ b/api/src/main/java/org/apache/iceberg/Snapshot.java @@ -164,12 +164,22 @@ default Iterable removedDeleteFiles(FileIO io) { } /** - * Return the location of this snapshot's manifest list, or null if it is not separate. + * Return the location of this snapshot's manifest list, or null if this snapshot has no manifest + * list. * * @return the location of the manifest list for this Snapshot */ String manifestListLocation(); + /** + * Return the location of this snapshot's root manifest, or null if it has no root manifest. + * + * @return the location of the root manifest for this snapshot + */ + default String rootManifestLocation() { + return null; + } + /** * Return the id of the schema used when this snapshot was created, or null if this information is * not available. diff --git a/core/src/main/java/org/apache/iceberg/BaseSnapshot.java b/core/src/main/java/org/apache/iceberg/BaseSnapshot.java index 826b9624c0e6..bd311d1ffa62 100644 --- a/core/src/main/java/org/apache/iceberg/BaseSnapshot.java +++ b/core/src/main/java/org/apache/iceberg/BaseSnapshot.java @@ -39,6 +39,7 @@ class BaseSnapshot implements Snapshot { private final long sequenceNumber; private final long timestampMillis; private final String manifestListLocation; + private final String rootManifestLocation; private final String operation; private final Map summary; private final Integer schemaId; @@ -65,9 +66,18 @@ class BaseSnapshot implements Snapshot { Map summary, Integer schemaId, String manifestList, + String rootManifest, Long firstRowId, Long addedRows, String keyId) { + Preconditions.checkArgument( + manifestList == null || rootManifest == null, + "Snapshot %s cannot have both a manifest list and a root manifest", + snapshotId); + Preconditions.checkArgument( + manifestList != null || rootManifest != null, + "Snapshot %s must have a manifest list or a root manifest", + snapshotId); Preconditions.checkArgument( firstRowId == null || firstRowId >= 0, "Invalid first-row-id (cannot be negative): %s", @@ -87,6 +97,7 @@ class BaseSnapshot implements Snapshot { this.summary = summary; this.schemaId = schemaId; this.manifestListLocation = manifestList; + this.rootManifestLocation = rootManifest; this.v1ManifestLocations = null; this.firstRowId = firstRowId; this.addedRows = firstRowId != null ? addedRows : null; @@ -110,6 +121,7 @@ class BaseSnapshot implements Snapshot { this.summary = summary; this.schemaId = schemaId; this.manifestListLocation = null; + this.rootManifestLocation = null; this.v1ManifestLocations = v1ManifestLocations; this.firstRowId = null; this.addedRows = null; @@ -167,6 +179,11 @@ public String keyId() { } private void cacheManifests(FileIO fileIO) { + Preconditions.checkState( + rootManifestLocation == null, + "Cannot read manifests from snapshot %s: root manifests are not supported", + snapshotId); + if (fileIO == null) { throw new IllegalArgumentException("Cannot cache changes: FileIO is null"); } @@ -181,7 +198,7 @@ private void cacheManifests(FileIO fileIO) { } if (allManifests == null) { - // if manifests isn't set, then the snapshotFile is set and should be read to get the list + // if manifests isn't set, then the manifest list must be read this.allManifests = ManifestLists.read( ManifestLists.newInputFile( @@ -261,6 +278,11 @@ public String manifestListLocation() { return manifestListLocation; } + @Override + public String rootManifestLocation() { + return rootManifestLocation; + } + private void cacheDeleteFileChanges(FileIO fileIO) { Preconditions.checkArgument(fileIO != null, "Cannot cache delete file changes: FileIO is null"); @@ -370,6 +392,7 @@ public String toString() { .add("operation", operation) .add("summary", summary) .add("manifest-list", manifestListLocation) + .add("root-manifest", rootManifestLocation) .add("schema-id", schemaId) .add("first-row-id", firstRowId) .add("added-rows", addedRows) diff --git a/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java b/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java index e1afe80d661c..ef295bed51be 100644 --- a/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java +++ b/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java @@ -241,8 +241,8 @@ private static List updatePathInSnapshots( TableMetadata metadata, String sourcePrefix, String targetPrefix) { List newSnapshots = Lists.newArrayListWithCapacity(metadata.snapshots().size()); for (Snapshot snapshot : metadata.snapshots()) { - String newManifestListLocation = - newPath(snapshot.manifestListLocation(), sourcePrefix, targetPrefix); + String manifestList = snapshot.manifestListLocation(); + String rootManifest = snapshot.rootManifestLocation(); Snapshot newSnapshot = new BaseSnapshot( snapshot.sequenceNumber(), @@ -252,7 +252,8 @@ private static List updatePathInSnapshots( snapshot.operation(), snapshot.summary(), snapshot.schemaId(), - newManifestListLocation, + manifestList == null ? null : newPath(manifestList, sourcePrefix, targetPrefix), + rootManifest == null ? null : newPath(rootManifest, sourcePrefix, targetPrefix), snapshot.firstRowId(), snapshot.addedRows(), snapshot.keyId()); diff --git a/core/src/main/java/org/apache/iceberg/SnapshotParser.java b/core/src/main/java/org/apache/iceberg/SnapshotParser.java index 53cec16dcd87..29e56d9d7009 100644 --- a/core/src/main/java/org/apache/iceberg/SnapshotParser.java +++ b/core/src/main/java/org/apache/iceberg/SnapshotParser.java @@ -50,6 +50,7 @@ private SnapshotParser() {} private static final String OPERATION = "operation"; private static final String MANIFESTS = "manifests"; private static final String MANIFEST_LIST = "manifest-list"; + private static final String ROOT_MANIFEST = "root-manifest"; private static final String SCHEMA_ID = "schema-id"; private static final String FIRST_ROW_ID = "first-row-id"; private static final String ADDED_ROWS = "added-rows"; @@ -83,7 +84,10 @@ static void toJson(Snapshot snapshot, JsonGenerator generator) throws IOExceptio } String manifestList = snapshot.manifestListLocation(); - if (manifestList != null) { + String rootManifest = snapshot.rootManifestLocation(); + if (rootManifest != null) { + generator.writeStringField(ROOT_MANIFEST, rootManifest); + } else if (manifestList != null) { // write just the location. manifests should not be embedded in JSON along with a list generator.writeStringField(MANIFEST_LIST, manifestList); } else { @@ -176,9 +180,10 @@ static Snapshot fromJson(JsonNode node) { String keyId = JsonUtil.getStringOrNull(KEY_ID, node); - if (node.has(MANIFEST_LIST)) { - // the manifest list is stored in a manifest list file - String manifestList = JsonUtil.getString(MANIFEST_LIST, node); + String manifestList = JsonUtil.getStringOrNull(MANIFEST_LIST, node); + String rootManifest = JsonUtil.getStringOrNull(ROOT_MANIFEST, node); + + if (manifestList != null || rootManifest != null) { return new BaseSnapshot( sequenceNumber, snapshotId, @@ -188,6 +193,7 @@ static Snapshot fromJson(JsonNode node) { summary, schemaId, manifestList, + rootManifest, firstRowId, addedRows, keyId); diff --git a/core/src/main/java/org/apache/iceberg/SnapshotProducer.java b/core/src/main/java/org/apache/iceberg/SnapshotProducer.java index 93133d2fe56e..74c89b603c0b 100644 --- a/core/src/main/java/org/apache/iceberg/SnapshotProducer.java +++ b/core/src/main/java/org/apache/iceberg/SnapshotProducer.java @@ -372,6 +372,7 @@ public Snapshot apply() { summary(base), base.currentSchemaId(), manifestList.location(), + null, // rootManifest nextRowId, assignedRows, manifestListEncryptionKeys != null ? manifestListEncryptionKeys.fileKey().keyId() : null); diff --git a/core/src/test/java/org/apache/iceberg/TestDataTaskParser.java b/core/src/test/java/org/apache/iceberg/TestDataTaskParser.java index b85d14525b68..8e8361e68f43 100644 --- a/core/src/test/java/org/apache/iceberg/TestDataTaskParser.java +++ b/core/src/test/java/org/apache/iceberg/TestDataTaskParser.java @@ -219,6 +219,7 @@ private DataTask createDataTask() { summary1, 1, "file:/tmp/manifest1.avro", + null /* rootManifest */, null, null, null), @@ -231,6 +232,7 @@ private DataTask createDataTask() { summary2, 1, "file:/tmp/manifest2.avro", + null /* rootManifest */, null, null, null)); diff --git a/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java b/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java index c661ac834d45..4d36e985c5b9 100644 --- a/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java +++ b/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java @@ -381,6 +381,7 @@ public void testAddSnapshotToJson() throws IOException { ImmutableMap.of("files-added", "4", "files-deleted", "100"), schemaId, manifestList, + null /* rootManifest */, firstRowId, addedRows, "key-1"); @@ -414,6 +415,7 @@ public void testAddSnapshotFromJson() throws IOException { summary, schemaId, manifestList, + null /* rootManifest */, lastRowId, addedRows, "key-1"); @@ -423,6 +425,36 @@ public void testAddSnapshotFromJson() throws IOException { assertEquals(action, expected, MetadataUpdateParser.fromJson(json)); } + @Test + public void testAddSnapshotWithRootManifest() { + String action = MetadataUpdateParser.ADD_SNAPSHOT; + long snapshotId = 2; + String rootManifest = "root-manifest-2.parquet"; + + Snapshot snapshot = + new BaseSnapshot( + 0 /* sequenceNumber */, + snapshotId, + null /* parentId */, + System.currentTimeMillis(), + DataOperations.APPEND, + ImmutableMap.of("files-added", "4") /* summary */, + 3 /* schemaId */, + null /* manifestList */, + rootManifest, + 0L /* firstRowId */, + 10L /* addedRows */, + "key-1" /* keyId */); + String snapshotJson = SnapshotParser.toJson(snapshot, /* pretty */ false); + String json = String.format("{\"action\":\"%s\",\"snapshot\":%s}", action, snapshotJson); + assertThat(json).contains("\"root-manifest\":\"" + rootManifest + "\""); + assertThat(json).doesNotContain("manifest-list"); + + MetadataUpdate expected = new MetadataUpdate.AddSnapshot(snapshot); + assertThat(MetadataUpdateParser.toJson(expected)).isEqualTo(json); + assertEquals(action, expected, MetadataUpdateParser.fromJson(json)); + } + /** RemoveSnapshot * */ @Test public void testRemoveSnapshotFromJson() { @@ -1270,6 +1302,8 @@ private static void assertEqualsAddSnapshot( assertThat(actual.snapshot().snapshotId()).isEqualTo(expected.snapshot().snapshotId()); assertThat(actual.snapshot().manifestListLocation()) .isEqualTo(expected.snapshot().manifestListLocation()); + assertThat(actual.snapshot().rootManifestLocation()) + .isEqualTo(expected.snapshot().rootManifestLocation()); assertThat(actual.snapshot().summary()) .as("Snapshot summary should be equivalent") .containsExactlyEntriesOf(expected.snapshot().summary()); diff --git a/core/src/test/java/org/apache/iceberg/TestRewriteTablePathUtil.java b/core/src/test/java/org/apache/iceberg/TestRewriteTablePathUtil.java index 64429dc3c12b..38ecf3509455 100644 --- a/core/src/test/java/org/apache/iceberg/TestRewriteTablePathUtil.java +++ b/core/src/test/java/org/apache/iceberg/TestRewriteTablePathUtil.java @@ -303,6 +303,40 @@ public void testReplacePathsWithTrailingSeparatorInPrefix() { assertThat(replaced.location()).isEqualTo("s3://bucket/restored/table"); } + @Test + void replacePathsRewritesRootManifestLocation() { + String rootManifestPath = "/metadata/root-manifest.parquet"; + String sourcePrefix = "s3://bucket/warehouse/table"; + String targetPrefix = "s3://bucket/restored/table"; + String sourceRootManifest = sourcePrefix + rootManifestPath; + String targetRootManifest = targetPrefix + rootManifestPath; + TableMetadata metadata = + TableMetadata.newTableMetadata( + SCHEMA, PartitionSpec.unpartitioned(), sourcePrefix, ImmutableMap.of()); + Snapshot snapshot = + new BaseSnapshot( + 1 /* sequenceNumber */, + 1L /* snapshotId */, + null /* parentId */, + System.currentTimeMillis(), + DataOperations.APPEND, + null /* summary */, + metadata.currentSchemaId(), + null /* manifestList */, + sourceRootManifest, + null /* firstRowId */, + null /* addedRows */, + null /* keyId */); + TableMetadata withRootManifest = + TableMetadata.buildFrom(metadata).addSnapshot(snapshot).build(); + + TableMetadata replaced = + RewriteTablePathUtil.replacePaths(withRootManifest, sourcePrefix, targetPrefix); + + assertThat(replaced.snapshot(snapshot.snapshotId()).rootManifestLocation()) + .isEqualTo(targetRootManifest); + } + @Test public void testNewPathTableRename() { // Rename /tableX to /table (target is substring of source name) diff --git a/core/src/test/java/org/apache/iceberg/TestRowLineageMetadata.java b/core/src/test/java/org/apache/iceberg/TestRowLineageMetadata.java index 4d861d5a815c..7b490b08560f 100644 --- a/core/src/test/java/org/apache/iceberg/TestRowLineageMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestRowLineageMetadata.java @@ -70,40 +70,53 @@ public void cleanup() { public void testSnapshotRowIDValidation() { Snapshot snapshot = new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, null, null); + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, null, null, null); assertThat(snapshot.firstRowId()).isNull(); assertThat(snapshot.addedRows()).isNull(); // added-rows will be set to null if first-row-id is null snapshot = - new BaseSnapshot(0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, 10L, null); + new BaseSnapshot( + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, null, 10L, null); assertThat(snapshot.firstRowId()).isNull(); assertThat(snapshot.addedRows()).isNull(); // added-rows and first-row-id can be 0 snapshot = - new BaseSnapshot(0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 0L, 0L, null); + new BaseSnapshot( + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, 0L, 0L, null); assertThat(snapshot.firstRowId()).isEqualTo(0); assertThat(snapshot.addedRows()).isEqualTo(0); assertThatThrownBy( () -> new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, null, null)) + 0, + 1, + null, + 0, + DataOperations.APPEND, + null, + 1, + "ml.avro", + null, + 10L, + null, + null)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Invalid added-rows (required when first-row-id is set): null"); assertThatThrownBy( () -> new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 0L, -1L, null)) + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, 0L, -1L, null)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Invalid added-rows (cannot be negative): -1"); assertThatThrownBy( () -> new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", -1L, 1L, null)) + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, -1L, 1L, null)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Invalid first-row-id (cannot be negative): -1"); } @@ -116,7 +129,18 @@ public void testSnapshotAddition() { Snapshot addRows = new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "foo", base.nextRowId(), newRows, null); + 0, + 1, + null, + 0, + DataOperations.APPEND, + null, + 1, + "foo", + null, + base.nextRowId(), + newRows, + null); TableMetadata firstAddition = TableMetadata.buildFrom(base).addSnapshot(addRows).build(); @@ -132,6 +156,7 @@ public void testSnapshotAddition() { null, 1, "foo", + null /* rootManifest */, firstAddition.nextRowId(), newRows, null); @@ -149,7 +174,8 @@ public void testInvalidSnapshotAddition() { TableMetadata base = baseMetadata(); Snapshot invalidLastRow = - new BaseSnapshot(0, 1, null, 0, DataOperations.APPEND, null, 1, "foo", null, newRows, null); + new BaseSnapshot( + 0, 1, null, 0, DataOperations.APPEND, null, 1, "foo", null, null, newRows, null); assertThatThrownBy(() -> TableMetadata.buildFrom(base).addSnapshot(invalidLastRow)) .isInstanceOf(ValidationException.class) @@ -158,12 +184,34 @@ public void testInvalidSnapshotAddition() { // add rows to check TableMetadata validation; Snapshot rejects negative next-row-id Snapshot addRows = new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "foo", base.nextRowId(), newRows, null); + 0, + 1, + null, + 0, + DataOperations.APPEND, + null, + 1, + "foo", + null, + base.nextRowId(), + newRows, + null); TableMetadata added = TableMetadata.buildFrom(base).addSnapshot(addRows).build(); Snapshot invalidNewRows = new BaseSnapshot( - 1, 2, 1L, 1, DataOperations.APPEND, null, 1, "foo", added.nextRowId() - 1, 10L, null); + 1, + 2, + 1L, + 1, + DataOperations.APPEND, + null, + 1, + "foo", + null, + added.nextRowId() - 1, + 10L, + null); assertThatThrownBy(() -> TableMetadata.buildFrom(added).addSnapshot(invalidNewRows)) .isInstanceOf(ValidationException.class) diff --git a/core/src/test/java/org/apache/iceberg/TestSnapshot.java b/core/src/test/java/org/apache/iceberg/TestSnapshot.java index 398bce3e94d9..0f5a3f431766 100644 --- a/core/src/test/java/org/apache/iceberg/TestSnapshot.java +++ b/core/src/test/java/org/apache/iceberg/TestSnapshot.java @@ -19,6 +19,7 @@ package org.apache.iceberg; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assumptions.assumeThat; import org.apache.iceberg.expressions.Expressions; @@ -34,22 +35,22 @@ public class TestSnapshot extends TestBase { public void snapshotValueMethodsIncludeMetadataFields() { Snapshot snapshot = new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 5L, "key-1"); + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, 10L, 5L, "key-1"); Snapshot same = new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 5L, "key-1"); + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, 10L, 5L, "key-1"); Snapshot differentFirstRowId = new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 11L, 5L, "key-1"); + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, 11L, 5L, "key-1"); Snapshot differentAddedRows = new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 6L, "key-1"); + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, 10L, 6L, "key-1"); Snapshot differentKeyId = new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 5L, "key-2"); + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, 10L, 5L, "key-2"); Snapshot withoutMetadataFields = new BaseSnapshot( - 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, null, null); + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, null, null, null); assertThat(snapshot).isEqualTo(same); assertThat(snapshot.hashCode()).isEqualTo(same.hashCode()); @@ -60,6 +61,91 @@ public void snapshotValueMethodsIncludeMetadataFields() { assertThat(snapshot.toString()).contains("first-row-id=10", "added-rows=5", "key-id=key-1"); } + @TestTemplate + void snapshotWithRootManifestExposesLocation() { + Snapshot snapshot = + new BaseSnapshot( + 0 /* sequenceNumber */, + 1 /* snapshotId */, + null /* parentId */, + 0 /* timestampMillis */, + DataOperations.APPEND, + null /* summary */, + 1 /* schemaId */, + null /* manifestList */, + "root-manifest.parquet" /* rootManifest */, + 10L /* firstRowId */, + 5L /* addedRows */, + "key-1" /* keyId */); + + assertThat(snapshot.rootManifestLocation()).isEqualTo("root-manifest.parquet"); + assertThat(snapshot.manifestListLocation()).isNull(); + } + + @TestTemplate + void rejectsBothManifestListAndRootManifest() { + assertThatThrownBy( + () -> + new BaseSnapshot( + 0 /* sequenceNumber */, + 1 /* snapshotId */, + null /* parentId */, + 0 /* timestampMillis */, + DataOperations.APPEND, + null /* summary */, + 1 /* schemaId */, + "ml.avro" /* manifestList */, + "root-manifest.parquet" /* rootManifest */, + 10L /* firstRowId */, + 5L /* addedRows */, + "key-1" /* keyId */)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("cannot have both a manifest list and a root manifest"); + } + + @TestTemplate + void rejectsNeitherManifestListNorRootManifest() { + assertThatThrownBy( + () -> + new BaseSnapshot( + 0 /* sequenceNumber */, + 1 /* snapshotId */, + null /* parentId */, + 0 /* timestampMillis */, + DataOperations.APPEND, + null /* summary */, + 1 /* schemaId */, + null /* manifestList */, + null /* rootManifest */, + 10L /* firstRowId */, + 5L /* addedRows */, + "key-1" /* keyId */)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("must have a manifest list or a root manifest"); + } + + @TestTemplate + void allManifestsRejectsRootManifest() { + Snapshot snapshot = + new BaseSnapshot( + 0 /* sequenceNumber */, + 1 /* snapshotId */, + null /* parentId */, + 0 /* timestampMillis */, + DataOperations.APPEND, + null /* summary */, + 1 /* schemaId */, + null /* manifestList */, + "root-manifest.parquet" /* rootManifest */, + 10L /* firstRowId */, + 5L /* addedRows */, + "key-1" /* keyId */); + + assertThatThrownBy(() -> snapshot.allManifests(table.io())) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("root manifests are not supported"); + } + @TestTemplate public void testAppendFilesFromTable() { table.newFastAppend().appendFile(FILE_A).appendFile(FILE_B).commit(); diff --git a/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java b/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java index f16e4feca82b..f715642df1c3 100644 --- a/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java +++ b/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java @@ -53,6 +53,7 @@ public void testJsonConversion() throws IOException { null, 1, manifestList, + null /* rootManifest */, null, null, keyId); @@ -66,6 +67,7 @@ public void testJsonConversion() throws IOException { assertThat(snapshot.firstRowId()).isNull(); assertThat(snapshot.addedRows()).isNull(); assertThat(snapshot.keyId()).isEqualTo(keyId); + assertThat(snapshot.rootManifestLocation()).isNull(); } @Test @@ -84,6 +86,7 @@ public void testJsonConversionWithoutSchemaId() throws IOException { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -91,6 +94,7 @@ public void testJsonConversionWithoutSchemaId() throws IOException { Snapshot snapshot = SnapshotParser.fromJson(json); assertThat(snapshot.snapshotId()).isEqualTo(expected.snapshotId()); + assertThat(snapshot.rootManifestLocation()).isNull(); assertThat(snapshot.allManifests(ops.io())).isEqualTo(expected.allManifests(ops.io())); assertThat(snapshot.operation()).isNull(); assertThat(snapshot.summary()).isNull(); @@ -116,6 +120,7 @@ public void testJsonConversionWithOperation() throws IOException { ImmutableMap.of("files-added", "4", "files-deleted", "100"), 3, manifestList, + null /* rootManifest */, null, null, null); @@ -130,6 +135,7 @@ public void testJsonConversionWithOperation() throws IOException { assertThat(snapshot.timestampMillis()).isEqualTo(expected.timestampMillis()); assertThat(snapshot.parentId()).isEqualTo(expected.parentId()); assertThat(snapshot.manifestListLocation()).isEqualTo(expected.manifestListLocation()); + assertThat(snapshot.rootManifestLocation()).isNull(); assertThat(snapshot.allManifests(ops.io())).isEqualTo(expected.allManifests(ops.io())); assertThat(snapshot.operation()).isEqualTo(expected.operation()); assertThat(snapshot.summary()).isEqualTo(expected.summary()); @@ -156,6 +162,7 @@ public void testJsonConversionWithRowLineage() throws IOException { null, null, manifestList, + null /* rootManifest */, firstRowId, addedRows, null); @@ -163,6 +170,7 @@ public void testJsonConversionWithRowLineage() throws IOException { Snapshot snapshot = SnapshotParser.fromJson(json); assertThat(snapshot.snapshotId()).isEqualTo(expected.snapshotId()); + assertThat(snapshot.rootManifestLocation()).isNull(); assertThat(snapshot.allManifests(ops.io())).isEqualTo(expected.allManifests(ops.io())); assertThat(snapshot.operation()).isNull(); assertThat(snapshot.summary()).isNull(); @@ -217,6 +225,7 @@ public void testJsonConversionWithV1Manifests() { assertThat(snapshot.timestampMillis()).isEqualTo(expected.timestampMillis()); assertThat(snapshot.parentId()).isEqualTo(expected.parentId()); assertThat(snapshot.manifestListLocation()).isEqualTo(expected.manifestListLocation()); + assertThat(snapshot.rootManifestLocation()).isNull(); assertThat(snapshot.allManifests(ops.io())).isEqualTo(expected.allManifests(ops.io())); assertThat(snapshot.operation()).isEqualTo(expected.operation()); assertThat(snapshot.summary()).isEqualTo(expected.summary()); @@ -320,4 +329,35 @@ public void testJsonConversionEmptySummary() { currentMs); assertThat(SnapshotParser.toJson(snap)).isEqualTo(expected); } + + @Test + public void testRootManifestRoundTrip() { + long timestampMillis = System.currentTimeMillis(); + Snapshot expected = + new BaseSnapshot( + 1 /* sequenceNumber */, + 23 /* snapshotId */, + null /* parentId */, + timestampMillis /* timestampMillis */, + "append" /* operation */, + ImmutableMap.of("operation", "append") /* summary */, + 1 /* schemaId */, + null /* manifestList */, + "root-manifest-23.parquet" /* rootManifest */, + 0L /* firstRowId */, + 10L /* addedRows */, + "key-1" /* keyId */); + + String json = SnapshotParser.toJson(expected, false); + assertThat(json).contains("\"root-manifest\":\"root-manifest-23.parquet\""); + assertThat(json).doesNotContain("manifest-list"); + + Snapshot snapshot = SnapshotParser.fromJson(json); + assertThat(snapshot.manifestListLocation()).isNull(); + assertThat(snapshot.rootManifestLocation()).isEqualTo(expected.rootManifestLocation()); + assertThat(snapshot.keyId()).isEqualTo("key-1"); + assertThat(snapshot.firstRowId()).isEqualTo(0L); + assertThat(snapshot.addedRows()).isEqualTo(10L); + assertThat(SnapshotParser.toJson(snapshot, false)).isEqualTo(json); + } } diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java index 02c23405033a..bdf94c87edd0 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java @@ -120,6 +120,7 @@ public void testJsonConversion() throws Exception { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -138,6 +139,7 @@ public void testJsonConversion() throws Exception { null, 7, manifestList, + null /* rootManifest */, null, null, null); @@ -265,6 +267,7 @@ public void testBackwardCompat() throws Exception { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -283,6 +286,7 @@ public void testBackwardCompat() throws Exception { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -372,6 +376,7 @@ public void testInvalidMainBranch() throws IOException { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -391,6 +396,7 @@ public void testInvalidMainBranch() throws IOException { null, 7, manifestList, + null /* rootManifest */, null, null, null); @@ -452,7 +458,18 @@ public void testMainWithoutCurrent() throws IOException { createManifestListWithManifestFile(snapshotId, null, "file:/tmp/manifest1.avro"); Snapshot snapshot = new BaseSnapshot( - 0, snapshotId, null, snapshotId, null, null, null, manifestList, null, null, null); + 0, + snapshotId, + null, + snapshotId, + null, + null, + null, + manifestList, + null, + null, + null, + null); Schema schema = new Schema(6, Types.NestedField.required(10, "x", Types.StringType.get())); @@ -590,6 +607,7 @@ public void testJsonWithPreviousMetadataLog() throws Exception { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -608,6 +626,7 @@ public void testJsonWithPreviousMetadataLog() throws Exception { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -670,6 +689,7 @@ public void testAddPreviousMetadataRemoveNone() throws IOException { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -688,6 +708,7 @@ public void testAddPreviousMetadataRemoveNone() throws IOException { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -767,6 +788,7 @@ public void testAddPreviousMetadataRemoveOne() throws IOException { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -785,6 +807,7 @@ public void testAddPreviousMetadataRemoveOne() throws IOException { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -877,6 +900,7 @@ public void testAddPreviousMetadataRemoveMultiple() throws IOException { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -895,6 +919,7 @@ public void testAddPreviousMetadataRemoveMultiple() throws IOException { null, null, manifestList, + null /* rootManifest */, null, null, null); @@ -2088,6 +2113,7 @@ public void testAddSnapshotWithStaleSequenceNumberIsRetryable() { null, null, "file:/s1.avro", + null /* rootManifest */, null, null, null); @@ -2104,6 +2130,7 @@ public void testAddSnapshotWithStaleSequenceNumberIsRetryable() { null, null, "file:/s2.avro", + null /* rootManifest */, null, null, null); @@ -2133,6 +2160,7 @@ public void testAddSnapshotWithStaleFirstRowIdIsRetryable() { null, null, "file:/s1.avro", + null /* rootManifest */, 0L, 5L, null); @@ -2142,7 +2170,18 @@ public void testAddSnapshotWithStaleFirstRowIdIsRetryable() { // A snapshot with firstRowId=0 is stale (0 < nextRowId=5) Snapshot staleSnapshot = new BaseSnapshot( - 2, 2L, 1L, System.currentTimeMillis(), null, null, null, "file:/s2.avro", 0L, 1L, null); + 2, + 2L, + 1L, + System.currentTimeMillis(), + null, + null, + null, + "file:/s2.avro", + null, + 0L, + 1L, + null); assertThatThrownBy(() -> TableMetadata.buildFrom(withS1).addSnapshot(staleSnapshot)) .isInstanceOf(RetryableValidationException.class) @@ -2165,6 +2204,7 @@ public void testSetRefRejectsTagForMainBranch() { null, null, "file:/s1.avro", + null /* rootManifest */, null, null, null); @@ -2198,11 +2238,12 @@ public void addSnapshotRejectsNonMonotonicTimestampAfterV4(int formatVersion) { ImmutableMap.of(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion))); Snapshot parent = - new BaseSnapshot(1, 1L, null, 1_000L, null, null, null, "file:/s1.avro", 0L, 0L, null); + new BaseSnapshot( + 1, 1L, null, 1_000L, null, null, null, "file:/s1.avro", null, 0L, 0L, null); TableMetadata withParent = TableMetadata.buildFrom(base).addSnapshot(parent).build(); Snapshot child = - new BaseSnapshot(2, 2L, 1L, 1_000L, null, null, null, "file:/s2.avro", 0L, 0L, null); + new BaseSnapshot(2, 2L, 1L, 1_000L, null, null, null, "file:/s2.avro", null, 0L, 0L, null); assertThatThrownBy(() -> TableMetadata.buildFrom(withParent).addSnapshot(child)) .isInstanceOf(ValidationException.class) @@ -2235,6 +2276,7 @@ public void addSnapshotAllowsNonMonotonicTimestampBeforeV4(int formatVersion) { null, null, "file:/s1.avro", + null /* rootManifest */, parentFirstRowId, parentAddedRows, null); @@ -2255,6 +2297,7 @@ public void addSnapshotAllowsNonMonotonicTimestampBeforeV4(int formatVersion) { null, null, "file:/s2.avro", + null /* rootManifest */, childFirstRowId, childAddedRows, null); @@ -2288,6 +2331,7 @@ public void snapshotTimestampMayBeAheadOfTableMetadataLastUpdatedMillis(int form null, null, "file:/s1.avro", + null /* rootManifest */, firstRowId, addedRows, null);