Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion api/src/main/java/org/apache/iceberg/Snapshot.java
Original file line number Diff line number Diff line change
Expand Up @@ -164,12 +164,22 @@ default Iterable<DeleteFile> removedDeleteFiles(FileIO io) {
}

/**
* Return the location of this snapshot's manifest list, or null if it is not separate.
* Return the location of this snapshot's manifest list, or null if this snapshot has no manifest
* list.
*
* @return the location of the manifest list for this Snapshot
*/
String manifestListLocation();

/**
* Return the location of this snapshot's root manifest, or null if it has no root manifest.
*
* @return the location of the root manifest for this snapshot
*/
default String rootManifestLocation() {
return null;
}

/**
* Return the id of the schema used when this snapshot was created, or null if this information is
* not available.
Expand Down
25 changes: 24 additions & 1 deletion core/src/main/java/org/apache/iceberg/BaseSnapshot.java
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ class BaseSnapshot implements Snapshot {
private final long sequenceNumber;
private final long timestampMillis;
private final String manifestListLocation;
private final String rootManifestLocation;
private final String operation;
private final Map<String, String> summary;
private final Integer schemaId;
Expand All @@ -65,9 +66,18 @@ class BaseSnapshot implements Snapshot {
Map<String, String> summary,
Integer schemaId,
String manifestList,
String rootManifest,

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I followed @gaborkaszab 's suggestion and dropped the list-only overload, keeping a single constructor that takes both locations. Existing callers pass null for rootManifest.

The upside is a single signature. The downside is every previous list-only call site now passes an extra null, which leads to churning in test code.

This is how BaseSnapshot has grown before: schemaId (#2275), firstRowId/addedRows (#11948), and keyId (#12927) were added to the same constructor.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We might want to introduce a dedicated contract once all the other V4 Snapshot fields are introduced, but let it be the concert of the follow-ups, not this one.
Thanks!

Long firstRowId,
Long addedRows,
String keyId) {
Preconditions.checkArgument(
Comment thread
rdblue marked this conversation as resolved.
Resolved
manifestList == null || rootManifest == null,
"Snapshot %s cannot have both a manifest list and a root manifest",
snapshotId);
Preconditions.checkArgument(
manifestList != null || rootManifest != null,
"Snapshot %s must have a manifest list or a root manifest",
snapshotId);
Preconditions.checkArgument(
firstRowId == null || firstRowId >= 0,
"Invalid first-row-id (cannot be negative): %s",
Expand All @@ -87,6 +97,7 @@ class BaseSnapshot implements Snapshot {
this.summary = summary;
this.schemaId = schemaId;
this.manifestListLocation = manifestList;
this.rootManifestLocation = rootManifest;
this.v1ManifestLocations = null;
this.firstRowId = firstRowId;
this.addedRows = firstRowId != null ? addedRows : null;
Expand All @@ -110,6 +121,7 @@ class BaseSnapshot implements Snapshot {
this.summary = summary;
this.schemaId = schemaId;
this.manifestListLocation = null;
this.rootManifestLocation = null;
this.v1ManifestLocations = v1ManifestLocations;
this.firstRowId = null;
this.addedRows = null;
Expand Down Expand Up @@ -167,6 +179,11 @@ public String keyId() {
}

private void cacheManifests(FileIO fileIO) {
Preconditions.checkState(
rootManifestLocation == null,
"Cannot read manifests from snapshot %s: root manifests are not supported",
snapshotId);

if (fileIO == null) {
throw new IllegalArgumentException("Cannot cache changes: FileIO is null");
}
Expand All @@ -181,7 +198,7 @@ private void cacheManifests(FileIO fileIO) {
}

if (allManifests == null) {
// if manifests isn't set, then the snapshotFile is set and should be read to get the list
// if manifests isn't set, then the manifest list must be read
this.allManifests =
ManifestLists.read(
ManifestLists.newInputFile(
Expand Down Expand Up @@ -261,6 +278,11 @@ public String manifestListLocation() {
return manifestListLocation;
}

@Override
public String rootManifestLocation() {
return rootManifestLocation;
}

private void cacheDeleteFileChanges(FileIO fileIO) {
Preconditions.checkArgument(fileIO != null, "Cannot cache delete file changes: FileIO is null");

Expand Down Expand Up @@ -370,6 +392,7 @@ public String toString() {
.add("operation", operation)
.add("summary", summary)
.add("manifest-list", manifestListLocation)
.add("root-manifest", rootManifestLocation)
.add("schema-id", schemaId)
.add("first-row-id", firstRowId)
.add("added-rows", addedRows)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -241,8 +241,8 @@ private static List<Snapshot> updatePathInSnapshots(
TableMetadata metadata, String sourcePrefix, String targetPrefix) {
List<Snapshot> newSnapshots = Lists.newArrayListWithCapacity(metadata.snapshots().size());
for (Snapshot snapshot : metadata.snapshots()) {
String newManifestListLocation =

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

there are a few more places in this class that reference snapshot.manifestListLocation() directly, should we update those as well?

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think that is necessary. This is updating this method to fulfill its contract, not going down the rabbit hole of making relocation work in v4.

newPath(snapshot.manifestListLocation(), sourcePrefix, targetPrefix);
String manifestList = snapshot.manifestListLocation();

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do we also need to handle ReachableFileCleanup and IncrementalFileCleanup?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd keep that out of this PR since nothing writes root manifests yet. Both cleanup paths read through allManifests, which now throws UnsupportedOperationException for root-manifest snapshots, so they fail loudly instead of silently skipping files.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We're introducing the new field to Snapshot, I don't think that we should implement table rewrite functionality right away. Could that be separate?

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it's reasonable to include this. This behavior is correct for this method's contract. We don't need to make the whole table rewrite work, but where we can update methods to fully implement their guarantees it seems like a good idea.

String rootManifest = snapshot.rootManifestLocation();
Snapshot newSnapshot =
new BaseSnapshot(
snapshot.sequenceNumber(),
Expand All @@ -252,7 +252,8 @@ private static List<Snapshot> updatePathInSnapshots(
snapshot.operation(),
snapshot.summary(),
snapshot.schemaId(),
newManifestListLocation,
manifestList == null ? null : newPath(manifestList, sourcePrefix, targetPrefix),
rootManifest == null ? null : newPath(rootManifest, sourcePrefix, targetPrefix),

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we should probably have a test in TestRewriteTablePathUtil that makes sure the root manifest is rewritten to the new location

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

added replacePathsRewritesRootManifestLocation, which asserts the rewritten location.

snapshot.firstRowId(),
snapshot.addedRows(),
snapshot.keyId());
Expand Down
14 changes: 10 additions & 4 deletions core/src/main/java/org/apache/iceberg/SnapshotParser.java
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ private SnapshotParser() {}
private static final String OPERATION = "operation";
private static final String MANIFESTS = "manifests";
private static final String MANIFEST_LIST = "manifest-list";
private static final String ROOT_MANIFEST = "root-manifest";
private static final String SCHEMA_ID = "schema-id";
private static final String FIRST_ROW_ID = "first-row-id";
private static final String ADDED_ROWS = "added-rows";
Expand Down Expand Up @@ -83,7 +84,10 @@ static void toJson(Snapshot snapshot, JsonGenerator generator) throws IOExceptio
}

String manifestList = snapshot.manifestListLocation();
if (manifestList != null) {
String rootManifest = snapshot.rootManifestLocation();
if (rootManifest != null) {
generator.writeStringField(ROOT_MANIFEST, rootManifest);
} else if (manifestList != null) {
// write just the location. manifests should not be embedded in JSON along with a list
generator.writeStringField(MANIFEST_LIST, manifestList);
} else {
Expand Down Expand Up @@ -176,9 +180,10 @@ static Snapshot fromJson(JsonNode node) {

String keyId = JsonUtil.getStringOrNull(KEY_ID, node);

if (node.has(MANIFEST_LIST)) {
// the manifest list is stored in a manifest list file
String manifestList = JsonUtil.getString(MANIFEST_LIST, node);
String manifestList = JsonUtil.getStringOrNull(MANIFEST_LIST, node);
String rootManifest = JsonUtil.getStringOrNull(ROOT_MANIFEST, node);

if (manifestList != null || rootManifest != null) {
return new BaseSnapshot(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This seems another reason why we should drop the manifest list only constructor.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, with the list-only constructor gone, fromJson now has a single call that passes both manifestList and rootManifest. The v1 embedded-manifests constructor is still separate.

sequenceNumber,
snapshotId,
Expand All @@ -188,6 +193,7 @@ static Snapshot fromJson(JsonNode node) {
summary,
schemaId,
manifestList,
rootManifest,
firstRowId,
addedRows,
keyId);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,7 @@ public Snapshot apply() {
summary(base),
base.currentSchemaId(),
manifestList.location(),
null, // rootManifest
nextRowId,
assignedRows,
manifestListEncryptionKeys != null ? manifestListEncryptionKeys.fileKey().keyId() : null);
Expand Down
2 changes: 2 additions & 0 deletions core/src/test/java/org/apache/iceberg/TestDataTaskParser.java
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,7 @@ private DataTask createDataTask() {
summary1,
1,
"file:/tmp/manifest1.avro",
null /* rootManifest */,
null,
null,
null),
Expand All @@ -231,6 +232,7 @@ private DataTask createDataTask() {
summary2,
1,
"file:/tmp/manifest2.avro",
null /* rootManifest */,
null,
null,
null));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -381,6 +381,7 @@ public void testAddSnapshotToJson() throws IOException {
ImmutableMap.of("files-added", "4", "files-deleted", "100"),
schemaId,
manifestList,
null /* rootManifest */,
firstRowId,
addedRows,
"key-1");
Expand Down Expand Up @@ -414,6 +415,7 @@ public void testAddSnapshotFromJson() throws IOException {
summary,
schemaId,
manifestList,
null /* rootManifest */,
lastRowId,
addedRows,
"key-1");
Expand All @@ -423,6 +425,36 @@ public void testAddSnapshotFromJson() throws IOException {
assertEquals(action, expected, MetadataUpdateParser.fromJson(json));
}

@Test
public void testAddSnapshotWithRootManifest() {
String action = MetadataUpdateParser.ADD_SNAPSHOT;
long snapshotId = 2;
String rootManifest = "root-manifest-2.parquet";

Snapshot snapshot =
new BaseSnapshot(
0 /* sequenceNumber */,
snapshotId,
null /* parentId */,
System.currentTimeMillis(),
DataOperations.APPEND,
ImmutableMap.of("files-added", "4") /* summary */,
3 /* schemaId */,
null /* manifestList */,
rootManifest,
0L /* firstRowId */,
10L /* addedRows */,
"key-1" /* keyId */);
String snapshotJson = SnapshotParser.toJson(snapshot, /* pretty */ false);
String json = String.format("{\"action\":\"%s\",\"snapshot\":%s}", action, snapshotJson);
assertThat(json).contains("\"root-manifest\":\"" + rootManifest + "\"");
assertThat(json).doesNotContain("manifest-list");

MetadataUpdate expected = new MetadataUpdate.AddSnapshot(snapshot);
assertThat(MetadataUpdateParser.toJson(expected)).isEqualTo(json);
assertEquals(action, expected, MetadataUpdateParser.fromJson(json));
}

/** RemoveSnapshot * */
@Test
public void testRemoveSnapshotFromJson() {
Expand Down Expand Up @@ -1270,6 +1302,8 @@ private static void assertEqualsAddSnapshot(
assertThat(actual.snapshot().snapshotId()).isEqualTo(expected.snapshot().snapshotId());
assertThat(actual.snapshot().manifestListLocation())
.isEqualTo(expected.snapshot().manifestListLocation());
assertThat(actual.snapshot().rootManifestLocation())
.isEqualTo(expected.snapshot().rootManifestLocation());
assertThat(actual.snapshot().summary())
.as("Snapshot summary should be equivalent")
.containsExactlyEntriesOf(expected.snapshot().summary());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,40 @@ public void testReplacePathsWithTrailingSeparatorInPrefix() {
assertThat(replaced.location()).isEqualTo("s3://bucket/restored/table");
}

@Test
void replacePathsRewritesRootManifestLocation() {
String rootManifestPath = "/metadata/root-manifest.parquet";
String sourcePrefix = "s3://bucket/warehouse/table";
String targetPrefix = "s3://bucket/restored/table";
String sourceRootManifest = sourcePrefix + rootManifestPath;
String targetRootManifest = targetPrefix + rootManifestPath;
TableMetadata metadata =
TableMetadata.newTableMetadata(
SCHEMA, PartitionSpec.unpartitioned(), sourcePrefix, ImmutableMap.of());
Snapshot snapshot =
new BaseSnapshot(
1 /* sequenceNumber */,
1L /* snapshotId */,
null /* parentId */,
System.currentTimeMillis(),
DataOperations.APPEND,
null /* summary */,
metadata.currentSchemaId(),
null /* manifestList */,
sourceRootManifest,
null /* firstRowId */,
null /* addedRows */,
null /* keyId */);
TableMetadata withRootManifest =
TableMetadata.buildFrom(metadata).addSnapshot(snapshot).build();

TableMetadata replaced =
RewriteTablePathUtil.replacePaths(withRootManifest, sourcePrefix, targetPrefix);

assertThat(replaced.snapshot(snapshot.snapshotId()).rootManifestLocation())
.isEqualTo(targetRootManifest);
}

@Test
public void testNewPathTableRename() {
// Rename /tableX to /table (target is substring of source name)
Expand Down
Loading
Loading