From e550cafd522dafa8634ac88916da0347926b19b7 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 16 Nov 2026 15:27:41 -0800 Subject: [PATCH 1/3] Core, API: add snapshot timestamp tracking and _last_updated_timestamp metadata column V4 row lineage needs a last-updated timestamp that is inherited the same way as the last-updated sequence number. This adds the snapshot_timestamp tracking field to the root and leaf manifest schemas, the root manifest snapshot timestamp, the _last_updated_timestamp metadata column, and the ContentFile and ManifestFile accessors. Read-side inheritance and commit-side assignment follow in later changes. Generated-by: Cursor Co-authored-by: Cursor --- .../java/org/apache/iceberg/ContentFile.java | 13 ++ .../java/org/apache/iceberg/ManifestFile.java | 5 + .../org/apache/iceberg/MetadataColumns.java | 12 +- .../org/apache/iceberg/RootManifestFile.java | 19 +- .../java/org/apache/iceberg/StatsUtil.java | 6 +- .../apache/iceberg/TrackedFileAdapters.java | 21 ++ .../java/org/apache/iceberg/Tracking.java | 15 +- .../org/apache/iceberg/TrackingBuilder.java | 11 +- .../org/apache/iceberg/TrackingStruct.java | 22 +- .../java/org/apache/iceberg/V4Metadata.java | 60 ++++-- .../org/apache/iceberg/TestFilePlanner.java | 2 + .../apache/iceberg/TestRootManifestFile.java | 39 +++- .../org/apache/iceberg/TestStatsUtil.java | 8 +- .../iceberg/TestTrackedFileAdapters.java | 15 +- .../apache/iceberg/TestTrackingBuilder.java | 37 +++- .../apache/iceberg/TestTrackingStruct.java | 92 +++++++-- .../apache/iceberg/TestV4ManifestReader.java | 195 +++++++++++++----- .../apache/iceberg/TestV4ManifestWriter.java | 125 +++++++---- .../org/apache/iceberg/V4TestComparators.java | 1 + 19 files changed, 541 insertions(+), 157 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/ContentFile.java b/api/src/main/java/org/apache/iceberg/ContentFile.java index 47daba02cd3c..708b2d375d38 100644 --- a/api/src/main/java/org/apache/iceberg/ContentFile.java +++ b/api/src/main/java/org/apache/iceberg/ContentFile.java @@ -183,6 +183,19 @@ default Long fileSequenceNumber() { return null; } + /** + * Returns the microsecond timestamp of the snapshot that added or changed this file. + * + *

The value is the snapshot's {@code timestamp-ms} converted to microseconds and is assigned + * at commit. + * + *

Returns null if the snapshot timestamp is unknown. This may happen for files in tables with + * format version below 4. + */ + default Long snapshotTimestampMicros() { + return null; + } + /** * Returns the starting row ID to assign to new rows in the data file (with _row_id set to null). */ diff --git a/api/src/main/java/org/apache/iceberg/ManifestFile.java b/api/src/main/java/org/apache/iceberg/ManifestFile.java index 32c7adae679f..b03b6e6c0835 100644 --- a/api/src/main/java/org/apache/iceberg/ManifestFile.java +++ b/api/src/main/java/org/apache/iceberg/ManifestFile.java @@ -230,6 +230,11 @@ default Long firstRowId() { return null; } + /** Returns the microsecond timestamp of the snapshot that added the manifest file. */ + default Long snapshotTimestampMicros() { + return null; + } + /** Returns the manifest deletion vector, or null if absent. */ default ManifestBitmap manifestDeletionVector() { return null; diff --git a/core/src/main/java/org/apache/iceberg/MetadataColumns.java b/core/src/main/java/org/apache/iceberg/MetadataColumns.java index 7b2629007649..e10aca141e4d 100644 --- a/core/src/main/java/org/apache/iceberg/MetadataColumns.java +++ b/core/src/main/java/org/apache/iceberg/MetadataColumns.java @@ -106,6 +106,12 @@ private MetadataColumns() {} "_last_updated_sequence_number", Types.LongType.get(), "Sequence number when the row was last updated"); + public static final NestedField LAST_UPDATED_TIMESTAMP = + NestedField.optional( + Integer.MAX_VALUE - 109, + "_last_updated_timestamp", + Types.TimestampType.withZone(), + "Snapshot timestamp when the row was last updated"); private static final Map META_COLUMNS = ImmutableMap.of( @@ -114,7 +120,8 @@ private MetadataColumns() {} IS_DELETED.name(), IS_DELETED, SPEC_ID.name(), SPEC_ID, ROW_ID.name(), ROW_ID, - LAST_UPDATED_SEQUENCE_NUMBER.name(), LAST_UPDATED_SEQUENCE_NUMBER); + LAST_UPDATED_SEQUENCE_NUMBER.name(), LAST_UPDATED_SEQUENCE_NUMBER, + LAST_UPDATED_TIMESTAMP.name(), LAST_UPDATED_TIMESTAMP); private static final Set META_IDS = ImmutableSet.of( @@ -124,7 +131,8 @@ private MetadataColumns() {} SPEC_ID.fieldId(), PARTITION_COLUMN_ID, ROW_ID.fieldId(), - LAST_UPDATED_SEQUENCE_NUMBER.fieldId()); + LAST_UPDATED_SEQUENCE_NUMBER.fieldId(), + LAST_UPDATED_TIMESTAMP.fieldId()); public static Set metadataFieldIds() { return META_IDS; diff --git a/core/src/main/java/org/apache/iceberg/RootManifestFile.java b/core/src/main/java/org/apache/iceberg/RootManifestFile.java index 3c757d8d7895..fc6abf39758b 100644 --- a/core/src/main/java/org/apache/iceberg/RootManifestFile.java +++ b/core/src/main/java/org/apache/iceberg/RootManifestFile.java @@ -21,6 +21,7 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.concurrent.TimeUnit; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; @@ -42,6 +43,7 @@ static RootManifestFile from(Snapshot snapshot, String tableLocation, FileIO io) io.newInputFile(location).getLength(), snapshot.snapshotId(), snapshot.sequenceNumber(), + TimeUnit.MILLISECONDS.toMicros(snapshot.timestampMillis()), snapshot.firstRowId(), /* keyMetadata= */ null); } @@ -50,6 +52,7 @@ static RootManifestFile from(Snapshot snapshot, String tableLocation, FileIO io) private final long length; private final long snapshotId; private final long sequenceNumber; + private final long snapshotTimestampMicros; private final Long firstRowId; private final byte[] keyMetadata; @@ -58,6 +61,7 @@ static RootManifestFile from(Snapshot snapshot, String tableLocation, FileIO io) long length, long snapshotId, long sequenceNumber, + long snapshotTimestampMicros, Long firstRowId, ByteBuffer keyMetadata) { Preconditions.checkArgument(location != null, "Invalid location: null"); @@ -65,6 +69,7 @@ static RootManifestFile from(Snapshot snapshot, String tableLocation, FileIO io) this.length = length; this.snapshotId = snapshotId; this.sequenceNumber = sequenceNumber; + this.snapshotTimestampMicros = snapshotTimestampMicros; this.firstRowId = firstRowId; this.keyMetadata = ByteBuffers.toByteArray(keyMetadata); } @@ -144,6 +149,11 @@ public ByteBuffer keyMetadata() { return keyMetadata == null ? null : ByteBuffer.wrap(keyMetadata); } + @Override + public Long snapshotTimestampMicros() { + return snapshotTimestampMicros; + } + @Override public Long firstRowId() { return firstRowId; @@ -161,7 +171,13 @@ public ManifestFile copy() { ? null : ByteBuffer.wrap(Arrays.copyOf(keyMetadata, keyMetadata.length)); return new RootManifestFile( - location, length, snapshotId, sequenceNumber, firstRowId, keyMetadataCopy); + location, + length, + snapshotId, + sequenceNumber, + snapshotTimestampMicros, + firstRowId, + keyMetadataCopy); } @Override @@ -171,6 +187,7 @@ public String toString() { .add("length", length) .add("snapshot_id", snapshotId) .add("sequence_number", sequenceNumber) + .add("snapshot_timestamp_micros", snapshotTimestampMicros) .add("first_row_id", firstRowId) .add("key_metadata", keyMetadata == null ? "null" : "(redacted)") .toString(); diff --git a/core/src/main/java/org/apache/iceberg/StatsUtil.java b/core/src/main/java/org/apache/iceberg/StatsUtil.java index 0df5da32c2fd..f9dbd1fb509e 100644 --- a/core/src/main/java/org/apache/iceberg/StatsUtil.java +++ b/core/src/main/java/org/apache/iceberg/StatsUtil.java @@ -56,6 +56,7 @@ private StatsUtil() {} private static final int LAST_UPDATED_SEQ_NUM_BASE_ID = 9_000; private static final int ROW_ID_BASE_ID = 9_200; + private static final int LAST_UPDATED_TIMESTAMP_BASE_ID = 9_400; // Offsets used for individual stats columns static final int LOWER_BOUND_OFFSET = 1; @@ -83,7 +84,8 @@ private StatsUtil() {} private static final Map METADATA_ID_TO_BASE_ID = ImmutableMap.of( MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId(), LAST_UPDATED_SEQ_NUM_BASE_ID, - MetadataColumns.ROW_ID.fieldId(), ROW_ID_BASE_ID); + MetadataColumns.ROW_ID.fieldId(), ROW_ID_BASE_ID, + MetadataColumns.LAST_UPDATED_TIMESTAMP.fieldId(), LAST_UPDATED_TIMESTAMP_BASE_ID); /** * Return the base ID of the stats struct for the given field ID, or -1 if the ID is out of range. @@ -116,6 +118,8 @@ static int toFieldId(int statId) { return MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId(); } else if (inBaseIdRange(ROW_ID_BASE_ID, statId)) { return MetadataColumns.ROW_ID.fieldId(); + } else if (inBaseIdRange(LAST_UPDATED_TIMESTAMP_BASE_ID, statId)) { + return MetadataColumns.LAST_UPDATED_TIMESTAMP.fieldId(); } else { throw new UnsupportedOperationException("Unsupported metadata stats field ID: " + statId); } diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java index 26d86c158d73..5e6acb5a8fa0 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java @@ -147,6 +147,12 @@ public Long fileSequenceNumber() { Tracking tracking = tracking(); return tracking != null ? tracking.fileSequenceNumber() : null; } + + @Override + public Long snapshotTimestampMicros() { + Tracking tracking = tracking(); + return tracking != null ? tracking.snapshotTimestampMicros() : null; + } } /** @@ -533,6 +539,11 @@ public Long firstRowId() { return file.tracking() != null ? file.tracking().firstRowId() : null; } + @Override + public Long snapshotTimestampMicros() { + return file.tracking().snapshotTimestampMicros(); + } + @Override public ManifestBitmap manifestDeletionVector() { return file.manifestInfo().manifestDeletionVector(); @@ -957,6 +968,11 @@ public ByteBuffer replacedPositions() { return null; } + @Override + public Long snapshotTimestampMicros() { + return null; + } + @Override public String manifestLocation() { return entry.file().manifestLocation(); @@ -1023,6 +1039,11 @@ public ByteBuffer replacedPositions() { return null; } + @Override + public Long snapshotTimestampMicros() { + return manifest.snapshotTimestampMicros(); + } + @Override public String manifestLocation() { return null; diff --git a/core/src/main/java/org/apache/iceberg/Tracking.java b/core/src/main/java/org/apache/iceberg/Tracking.java index ff02bffc3b4f..b1134432f3e3 100644 --- a/core/src/main/java/org/apache/iceberg/Tracking.java +++ b/core/src/main/java/org/apache/iceberg/Tracking.java @@ -65,6 +65,12 @@ interface Tracking { "replaced_positions", Types.BinaryType.get(), "Bitmap of positions replaced in this snapshot"); + Types.NestedField SNAPSHOT_TIMESTAMP = + Types.NestedField.optional( + 8, + "snapshot_timestamp", + Types.TimestampType.withZone(), + "Timestamp in microseconds of the snapshot that added or last modified the file's data"); static Types.StructType schema() { return Types.StructType.of( @@ -75,7 +81,8 @@ static Types.StructType schema() { MODIFIED_SNAPSHOT_ID, FIRST_ROW_ID, DELETED_POSITIONS, - REPLACED_POSITIONS); + REPLACED_POSITIONS, + SNAPSHOT_TIMESTAMP); } /** Returns the status of the entry. */ @@ -107,6 +114,12 @@ default boolean isLive() { /** Returns the bitmap of positions replaced in this snapshot. */ ByteBuffer replacedPositions(); + /** + * Returns the timestamp in microseconds of the snapshot that added or last modified the file's + * data. + */ + Long snapshotTimestampMicros(); + /** Returns the manifest location this entry was read from, or null. */ String manifestLocation(); diff --git a/core/src/main/java/org/apache/iceberg/TrackingBuilder.java b/core/src/main/java/org/apache/iceberg/TrackingBuilder.java index 261b2fa3604f..5eb91bd19ca5 100644 --- a/core/src/main/java/org/apache/iceberg/TrackingBuilder.java +++ b/core/src/main/java/org/apache/iceberg/TrackingBuilder.java @@ -86,6 +86,7 @@ static Tracking replaced(Tracking source, long newSnapshotId) { private Long modifiedSnapshotId; private byte[] deletedPositions; private byte[] replacedPositions; + private Long snapshotTimestampMicros; private TrackingBuilder(Long newSnapshotId) { this.status = EntryStatus.ADDED; @@ -97,6 +98,7 @@ private TrackingBuilder(Long newSnapshotId) { this.modifiedSnapshotId = null; this.deletedPositions = null; this.replacedPositions = null; + this.snapshotTimestampMicros = null; } private TrackingBuilder(long newSnapshotId, long dataSequenceNumber) { @@ -116,6 +118,7 @@ private TrackingBuilder(Tracking source, long newSnapshotId) { this.modifiedSnapshotId = source.modifiedSnapshotId(); this.deletedPositions = null; this.replacedPositions = null; + this.snapshotTimestampMicros = source.snapshotTimestampMicros(); } /** Indicates that the DV has been updated for the new Tracking. */ @@ -145,6 +148,8 @@ TrackingBuilder columnFilesUpdated() { // dataSequenceNumber as writers are required to rewrite v2 equality and position deletes to DVs // when applying column update. this.dataSequenceNumber = null; + // Reset to null to inherit from the new snapshot timestamp because the file's data changed. + this.snapshotTimestampMicros = null; return this; } @@ -177,7 +182,8 @@ Tracking build() { modifiedSnapshotId, firstRowId, deletedPositions, - replacedPositions); + replacedPositions, + snapshotTimestampMicros); } private static Tracking terminal(EntryStatus to, Tracking source, long newSnapshotId) { @@ -191,7 +197,8 @@ private static Tracking terminal(EntryStatus to, Tracking source, long newSnapsh source.modifiedSnapshotId(), source.firstRowId(), null, - null); + null, + source.snapshotTimestampMicros()); } private static void validateSource(Tracking source) { diff --git a/core/src/main/java/org/apache/iceberg/TrackingStruct.java b/core/src/main/java/org/apache/iceberg/TrackingStruct.java index d06eae71c58d..95c3171b4a7e 100644 --- a/core/src/main/java/org/apache/iceberg/TrackingStruct.java +++ b/core/src/main/java/org/apache/iceberg/TrackingStruct.java @@ -39,6 +39,7 @@ class TrackingStruct extends SupportsIndexProjection implements Tracking, Serial Tracking.FIRST_ROW_ID, Tracking.DELETED_POSITIONS, Tracking.REPLACED_POSITIONS, + Tracking.SNAPSHOT_TIMESTAMP, MetadataColumns.FILE_PATH, MetadataColumns.ROW_POSITION); @@ -62,6 +63,7 @@ class TrackingStruct extends SupportsIndexProjection implements Tracking, Serial private Long firstRowId = null; private byte[] deletedPositions = null; private byte[] replacedPositions = null; + private Long snapshotTimestampMicros = null; // set by manifest readers, not written to manifests private String manifestLocation = null; @@ -92,6 +94,7 @@ private TrackingStruct(TrackingStruct toCopy) { toCopy.replacedPositions != null ? Arrays.copyOf(toCopy.replacedPositions, toCopy.replacedPositions.length) : null; + this.snapshotTimestampMicros = toCopy.snapshotTimestampMicros; this.manifestLocation = toCopy.manifestLocation; this.manifestPos = toCopy.manifestPos; } @@ -104,7 +107,8 @@ private TrackingStruct(TrackingStruct toCopy) { Long modifiedSnapshotId, Long firstRowId, byte[] deletedPositions, - byte[] replacedPositions) { + byte[] replacedPositions, + Long snapshotTimestampMicros) { super(BASE_TYPE.fields().size()); this.status = status; this.snapshotId = snapshotId; @@ -114,6 +118,7 @@ private TrackingStruct(TrackingStruct toCopy) { this.firstRowId = firstRowId; this.deletedPositions = deletedPositions; this.replacedPositions = replacedPositions; + this.snapshotTimestampMicros = snapshotTimestampMicros; } /** @@ -220,6 +225,11 @@ public ByteBuffer replacedPositions() { return replacedPositions != null ? ByteBuffer.wrap(replacedPositions) : null; } + @Override + public Long snapshotTimestampMicros() { + return snapshotTimestampMicros; + } + @Override public String manifestLocation() { return manifestLocation; @@ -259,8 +269,10 @@ private Object getByPos(int pos) { case 7: return replacedPositions(); case 8: - return manifestLocation; + return snapshotTimestampMicros; case 9: + return manifestLocation; + case 10: return manifestPos; default: throw new UnsupportedOperationException("Unknown field ordinal: " + pos); @@ -295,9 +307,12 @@ protected void internalSet(int pos, T value) { this.replacedPositions = ByteBuffers.toByteArray((ByteBuffer) value); break; case 8: - this.manifestLocation = (String) value; + this.snapshotTimestampMicros = (Long) value; break; case 9: + this.manifestLocation = (String) value; + break; + case 10: this.manifestPos = (long) value; break; default: @@ -316,6 +331,7 @@ public String toString() { .add("first_row_id", firstRowId) .add("deleted_positions", deletedPositions == null ? "null" : "(binary)") .add("replaced_positions", replacedPositions == null ? "null" : "(binary)") + .add("snapshot_timestamp_micros", snapshotTimestampMicros) .toString(); } } diff --git a/core/src/main/java/org/apache/iceberg/V4Metadata.java b/core/src/main/java/org/apache/iceberg/V4Metadata.java index cba9c02abc0b..ebfec4b4248c 100644 --- a/core/src/main/java/org/apache/iceberg/V4Metadata.java +++ b/core/src/main/java/org/apache/iceberg/V4Metadata.java @@ -403,7 +403,8 @@ private static void validateFirstRowId( Tracking.MODIFIED_SNAPSHOT_ID, Tracking.FIRST_ROW_ID, Tracking.DELETED_POSITIONS, - Tracking.REPLACED_POSITIONS); + Tracking.REPLACED_POSITIONS, + Tracking.SNAPSHOT_TIMESTAMP); private static final Types.StructType LEAF_TRACKING_TYPE = Types.StructType.of( @@ -412,10 +413,11 @@ private static void validateFirstRowId( Tracking.SEQUENCE_NUMBER, Tracking.FILE_SEQUENCE_NUMBER, Tracking.MODIFIED_SNAPSHOT_ID, - Tracking.FIRST_ROW_ID); + Tracking.FIRST_ROW_ID, + Tracking.SNAPSHOT_TIMESTAMP); /** Wrapper used to write {@link Tracking} using the v4 tracking schema. */ - private static class TrackingWrapper implements StructLike { + private abstract static class TrackingWrapper implements StructLike { private static final int SIZE = Tracking.schema().fields().size(); private final Long commitSnapshotId; @@ -477,28 +479,20 @@ public void set(int pos, T value) { @Override public T get(int pos, Class javaClass) { - return javaClass.cast(get(pos)); + return javaClass.cast(getInternal(pos)); } - private Object get(int pos) { - return switch (pos) { - case 0 -> wrapped.status().id(); - case 1 -> wrapped.snapshotId(); - case 2 -> sequenceNumber(wrapped.dataSequenceNumber()); - case 3 -> sequenceNumber(wrapped.fileSequenceNumber()); - case 4 -> wrapped.modifiedSnapshotId(); - case 5 -> firstRowId(); - case 6 -> wrapped.deletedPositions(); - case 7 -> wrapped.replacedPositions(); - default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - }; + protected abstract Object getInternal(int pos); + + protected Tracking wrapped() { + return wrapped; } - private Long sequenceNumber(Long sequenceNumber) { + protected Long sequenceNumber(Long sequenceNumber) { return sequenceNumber != null ? sequenceNumber : commitSequenceNumber; } - private Long firstRowId() { + protected Long firstRowId() { return assignedFirstRowId != null ? assignedFirstRowId : wrapped.firstRowId(); } } @@ -512,6 +506,20 @@ private TrackingWrapper wrap(Tracking tracking) { validateSequenceNumbers(tracking); return wrapInternal(tracking, null /* first row ID is inherited */); } + + @Override + protected Object getInternal(int pos) { + return switch (pos) { + case 0 -> wrapped().status().id(); + case 1 -> wrapped().snapshotId(); + case 2 -> sequenceNumber(wrapped().dataSequenceNumber()); + case 3 -> sequenceNumber(wrapped().fileSequenceNumber()); + case 4 -> wrapped().modifiedSnapshotId(); + case 5 -> firstRowId(); + case 6 -> wrapped().snapshotTimestampMicros(); + default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + }; + } } private static class RootTrackingWrapper extends TrackingWrapper { @@ -525,6 +533,22 @@ private TrackingWrapper wrap(Tracking tracking, Long firstRowId, String location validateSequenceNumbers(tracking); return wrapInternal(tracking, firstRowId); } + + @Override + protected Object getInternal(int pos) { + return switch (pos) { + case 0 -> wrapped().status().id(); + case 1 -> wrapped().snapshotId(); + case 2 -> sequenceNumber(wrapped().dataSequenceNumber()); + case 3 -> sequenceNumber(wrapped().fileSequenceNumber()); + case 4 -> wrapped().modifiedSnapshotId(); + case 5 -> firstRowId(); + case 6 -> wrapped().deletedPositions(); + case 7 -> wrapped().replacedPositions(); + case 8 -> wrapped().snapshotTimestampMicros(); + default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + }; + } } private static final Types.StructType MANIFEST_INFO_TYPE = diff --git a/core/src/test/java/org/apache/iceberg/TestFilePlanner.java b/core/src/test/java/org/apache/iceberg/TestFilePlanner.java index 4f50297b3831..c8052032bc9b 100644 --- a/core/src/test/java/org/apache/iceberg/TestFilePlanner.java +++ b/core/src/test/java/org/apache/iceberg/TestFilePlanner.java @@ -61,6 +61,7 @@ class TestFilePlanner { private static final long SEQUENCE_NUMBER = 7L; + private static final long SNAPSHOT_TIMESTAMP_MICROS = 1_700_000_000_123_000L; private static final long FIRST_ROW_ID = 1000L; private static final String TABLE_LOCATION = "s3://bucket/db/table"; private static final DeletionVector DV = @@ -819,6 +820,7 @@ private ManifestFile writeManifest( written.getLength(), SNAPSHOT_ID, SEQUENCE_NUMBER, + SNAPSHOT_TIMESTAMP_MICROS, FIRST_ROW_ID, /* keyMetadata= */ null); } diff --git a/core/src/test/java/org/apache/iceberg/TestRootManifestFile.java b/core/src/test/java/org/apache/iceberg/TestRootManifestFile.java index 9291b746e162..1160824edde0 100644 --- a/core/src/test/java/org/apache/iceberg/TestRootManifestFile.java +++ b/core/src/test/java/org/apache/iceberg/TestRootManifestFile.java @@ -30,6 +30,7 @@ class TestRootManifestFile { private static final long SNAPSHOT_ID = 42L; private static final long SEQUENCE_NUMBER = 7L; + private static final long SNAPSHOT_TIMESTAMP_MICROS = 1_700_000_000_123_000L; private static final long FIRST_ROW_ID = 1000L; private static final long LENGTH = 1024L; private static final String LOCATION = "s3://bucket/db/table/metadata/root.parquet"; @@ -39,12 +40,19 @@ void fieldAccess() { ByteBuffer keyMetadata = ByteBuffer.wrap(new byte[] {9, 8, 7}); RootManifestFile root = new RootManifestFile( - LOCATION, LENGTH, SNAPSHOT_ID, SEQUENCE_NUMBER, FIRST_ROW_ID, keyMetadata); + LOCATION, + LENGTH, + SNAPSHOT_ID, + SEQUENCE_NUMBER, + SNAPSHOT_TIMESTAMP_MICROS, + FIRST_ROW_ID, + keyMetadata); assertThat(root.path()).isEqualTo(LOCATION); assertThat(root.length()).isEqualTo(LENGTH); assertThat(root.snapshotId()).isEqualTo(SNAPSHOT_ID); assertThat(root.sequenceNumber()).isEqualTo(SEQUENCE_NUMBER); + assertThat(root.snapshotTimestampMicros()).isEqualTo(SNAPSHOT_TIMESTAMP_MICROS); assertThat(root.firstRowId()).isEqualTo(FIRST_ROW_ID); assertThat(root.keyMetadata()).isEqualTo(keyMetadata); } @@ -52,7 +60,14 @@ void fieldAccess() { @Test void defaultedFields() { RootManifestFile root = - new RootManifestFile(LOCATION, LENGTH, SNAPSHOT_ID, SEQUENCE_NUMBER, FIRST_ROW_ID, null); + new RootManifestFile( + LOCATION, + LENGTH, + SNAPSHOT_ID, + SEQUENCE_NUMBER, + SNAPSHOT_TIMESTAMP_MICROS, + FIRST_ROW_ID, + null); // keyMetadata assertThat(root.content()).isEqualTo(ManifestContent.DATA); assertThat(root.formatVersion()).isEqualTo(4); @@ -63,13 +78,20 @@ void copy() { ByteBuffer keyMetadata = ByteBuffer.wrap(new byte[] {9, 8, 7}); RootManifestFile original = new RootManifestFile( - LOCATION, LENGTH, SNAPSHOT_ID, SEQUENCE_NUMBER, FIRST_ROW_ID, keyMetadata); + LOCATION, + LENGTH, + SNAPSHOT_ID, + SEQUENCE_NUMBER, + SNAPSHOT_TIMESTAMP_MICROS, + FIRST_ROW_ID, + keyMetadata); ManifestFile copy = original.copy(); assertThat(copy.path()).isEqualTo(LOCATION); assertThat(copy.length()).isEqualTo(LENGTH); assertThat(copy.snapshotId()).isEqualTo(SNAPSHOT_ID); assertThat(copy.sequenceNumber()).isEqualTo(SEQUENCE_NUMBER); + assertThat(copy.snapshotTimestampMicros()).isEqualTo(SNAPSHOT_TIMESTAMP_MICROS); assertThat(copy.firstRowId()).isEqualTo(FIRST_ROW_ID); assertThat(copy.keyMetadata()).isEqualTo(keyMetadata); assertThat(copy.keyMetadata().array()) @@ -85,6 +107,7 @@ void toStringRedactsKeyMetadata() { LENGTH, SNAPSHOT_ID, SEQUENCE_NUMBER, + SNAPSHOT_TIMESTAMP_MICROS, FIRST_ROW_ID, ByteBuffer.wrap(new byte[] {9, 8, 7})); @@ -93,6 +116,7 @@ void toStringRedactsKeyMetadata() { .contains("length=" + LENGTH) .contains("snapshot_id=" + SNAPSHOT_ID) .contains("sequence_number=" + SEQUENCE_NUMBER) + .contains("snapshot_timestamp_micros=" + SNAPSHOT_TIMESTAMP_MICROS) .contains("first_row_id=" + FIRST_ROW_ID) .contains("key_metadata=(redacted)"); } @@ -100,7 +124,14 @@ void toStringRedactsKeyMetadata() { @Test void unsupportedAccessorsThrow() { RootManifestFile root = - new RootManifestFile(LOCATION, LENGTH, SNAPSHOT_ID, SEQUENCE_NUMBER, FIRST_ROW_ID, null); + new RootManifestFile( + LOCATION, + LENGTH, + SNAPSHOT_ID, + SEQUENCE_NUMBER, + SNAPSHOT_TIMESTAMP_MICROS, + FIRST_ROW_ID, + null); // keyMetadata assertThatThrownBy(root::partitionSpecId) .isInstanceOf(UnsupportedOperationException.class) diff --git a/core/src/test/java/org/apache/iceberg/TestStatsUtil.java b/core/src/test/java/org/apache/iceberg/TestStatsUtil.java index 419941fb42f3..09f29924cc54 100644 --- a/core/src/test/java/org/apache/iceberg/TestStatsUtil.java +++ b/core/src/test/java/org/apache/iceberg/TestStatsUtil.java @@ -52,6 +52,8 @@ public void testToBaseIdWithMetadata() { assertThat(StatsUtil.toBaseId(MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId())) .isEqualTo(9_000); assertThat(StatsUtil.toBaseId(MetadataColumns.ROW_ID.fieldId())).isEqualTo(9_200); + assertThat(StatsUtil.toBaseId(MetadataColumns.LAST_UPDATED_TIMESTAMP.fieldId())) + .isEqualTo(9_400); } @Test @@ -74,6 +76,8 @@ public void testToFieldIdWithMetadata() { assertThat(StatsUtil.toFieldId(9_000)) .isEqualTo(MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId()); assertThat(StatsUtil.toFieldId(9_200)).isEqualTo(MetadataColumns.ROW_ID.fieldId()); + assertThat(StatsUtil.toFieldId(9_400)) + .isEqualTo(MetadataColumns.LAST_UPDATED_TIMESTAMP.fieldId()); } @Test @@ -95,9 +99,9 @@ public void testToFieldIdOutsideRange() { @Test public void testToFieldIdReservedMetadataRange() { // 9,000 to 10,000 (exclusive) is reserved for metadata column stats - assertThatThrownBy(() -> StatsUtil.toFieldId(9_400)) + assertThatThrownBy(() -> StatsUtil.toFieldId(9_600)) .isInstanceOf(UnsupportedOperationException.class) - .hasMessage("Unsupported metadata stats field ID: 9400"); + .hasMessage("Unsupported metadata stats field ID: 9600"); } @Test diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java index 6e6e69ed72f1..1794bba376aa 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java @@ -51,6 +51,7 @@ class TestTrackedFileAdapters { private static final long DATA_SEQUENCE_NUMBER = 10L; private static final long FILE_SEQUENCE_NUMBER = 11L; private static final long FIRST_ROW_ID = 1000L; + private static final long SNAPSHOT_TIMESTAMP_MICROS = 1_700_000_000_123_456L; private static final int SORT_ORDER_ID = 3; private static final long MANIFEST_SEQUENCE_NUMBER = 5L; @@ -122,7 +123,8 @@ class TestTrackedFileAdapters { null, // modifiedSnapshotId FIRST_ROW_ID, null, // deletedPositions - null); // replacedPositions + null, // replacedPositions + SNAPSHOT_TIMESTAMP_MICROS); private static final ByteBuffer MANIFEST_KEY_METADATA = ByteBuffer.wrap(new byte[] {7, 8, 9}); @@ -194,7 +196,8 @@ void dataFileAdapterDelegation() { null, FIRST_ROW_ID, null, - null); + null, + SNAPSHOT_TIMESTAMP_MICROS); tracking.set(MANIFEST_LOCATION_ORDINAL, MANIFEST_LOCATION); tracking.set(MANIFEST_POSITION_ORDINAL, MANIFEST_POS); @@ -231,6 +234,7 @@ void dataFileAdapterDelegation() { assertThat(dataFile.sortOrderId()).isEqualTo(3); assertThat(dataFile.dataSequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER); assertThat(dataFile.fileSequenceNumber()).isEqualTo(FILE_SEQUENCE_NUMBER); + assertThat(dataFile.snapshotTimestampMicros()).isEqualTo(SNAPSHOT_TIMESTAMP_MICROS); assertThat(dataFile.firstRowId()).isEqualTo(FIRST_ROW_ID); assertThat(dataFile.keyMetadata()).isEqualTo(ByteBuffer.wrap(new byte[] {1, 2, 3})); assertThat(dataFile.splitOffsets()).containsExactly(50L, 100L); @@ -298,7 +302,8 @@ void dvDeleteFileAdapterDelegation() { 42L, FIRST_ROW_ID, null, - null); + null, + SNAPSHOT_TIMESTAMP_MICROS); tracking.set(MANIFEST_LOCATION_ORDINAL, MANIFEST_LOCATION); tracking.set(MANIFEST_POSITION_ORDINAL, MANIFEST_POS); @@ -342,6 +347,7 @@ void dvDeleteFileAdapterDelegation() { assertThat(dvFile.partition()).isSameAs(PARTITION); assertThat(dvFile.dataSequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER); assertThat(dvFile.fileSequenceNumber()).isEqualTo(FILE_SEQUENCE_NUMBER); + assertThat(dvFile.snapshotTimestampMicros()).isEqualTo(SNAPSHOT_TIMESTAMP_MICROS); assertThat(dvFile.manifestLocation()).isEqualTo(MANIFEST_LOCATION); // fields that are null for DVs @@ -388,6 +394,7 @@ private static void assertSameDataFile(DataFile actual, DataFile expected) { assertThat(actual.sortOrderId()).isEqualTo(expected.sortOrderId()); assertThat(actual.dataSequenceNumber()).isEqualTo(expected.dataSequenceNumber()); assertThat(actual.fileSequenceNumber()).isEqualTo(expected.fileSequenceNumber()); + assertThat(actual.snapshotTimestampMicros()).isEqualTo(expected.snapshotTimestampMicros()); assertThat(actual.firstRowId()).isEqualTo(expected.firstRowId()); assertThat(actual.pos()).isEqualTo(expected.pos()); assertThat(actual.manifestLocation()).isEqualTo(expected.manifestLocation()); @@ -423,6 +430,7 @@ private static TrackedFile serializableDataEntry() { 42L, FIRST_ROW_ID, null, + null, null); tracking.set(MANIFEST_LOCATION_ORDINAL, MANIFEST_LOCATION); tracking.set(MANIFEST_POSITION_ORDINAL, MANIFEST_POS); @@ -519,6 +527,7 @@ void manifestFileAdapterDelegation(FileContent contentType) { assertThat(manifest.modifiedFilesCount()).isEqualTo(MANIFEST_INFO.modifiedFilesCount()); assertThat(manifest.modifiedRowsCount()).isEqualTo(MANIFEST_INFO.modifiedRowsCount()); assertThat(manifest.firstRowId()).isEqualTo(FIRST_ROW_ID); + assertThat(manifest.snapshotTimestampMicros()).isEqualTo(SNAPSHOT_TIMESTAMP_MICROS); assertThat(manifest.keyMetadata()).isEqualTo(MANIFEST_KEY_METADATA); assertThat(manifest.manifestDeletionVector().buffer()) .isEqualTo(ByteBuffer.wrap(MumblingTestUtil.onlyFirstBitSetBytes())); diff --git a/core/src/test/java/org/apache/iceberg/TestTrackingBuilder.java b/core/src/test/java/org/apache/iceberg/TestTrackingBuilder.java index 7f1d1ca4cc44..9739a8709acf 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackingBuilder.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackingBuilder.java @@ -33,11 +33,14 @@ class TestTrackingBuilder { // Tests aren't expected to change these fields private static final Tracking SOURCE_TRACKING_ADDED = - new TrackingStruct(EntryStatus.ADDED, 42L, 10L, 10L, 43L, 1000L, null, null); + new TrackingStruct( + EntryStatus.ADDED, 42L, 10L, 10L, 43L, 1000L, null, null, 1_700_000_000_123_456L); private static final Tracking SOURCE_TRACKING_EXISTING = - new TrackingStruct(EntryStatus.EXISTING, 42L, 10L, 10L, 43L, 1000L, null, null); + new TrackingStruct( + EntryStatus.EXISTING, 42L, 10L, 10L, 43L, 1000L, null, null, 1_700_000_000_123_456L); private static final Tracking SOURCE_TRACKING_MODIFIED = - new TrackingStruct(EntryStatus.MODIFIED, 42L, 10L, 10L, 43L, 1000L, null, null); + new TrackingStruct( + EntryStatus.MODIFIED, 42L, 10L, 10L, 43L, 1000L, null, null, 1_700_000_000_123_456L); @Test void addedWithSameCommitDvStaysAdded() { @@ -64,6 +67,8 @@ void existingBuilderPreservesSourceFields() { assertThat(existing.fileSequenceNumber()).isEqualTo(SOURCE_TRACKING_ADDED.fileSequenceNumber()); assertThat(existing.modifiedSnapshotId()).isEqualTo(SOURCE_TRACKING_ADDED.modifiedSnapshotId()); assertThat(existing.firstRowId()).isEqualTo(SOURCE_TRACKING_ADDED.firstRowId()); + assertThat(existing.snapshotTimestampMicros()) + .isEqualTo(SOURCE_TRACKING_ADDED.snapshotTimestampMicros()); } @Test @@ -76,6 +81,8 @@ void deleteUpdatesSnapshotIdAndPreservesRest() { assertThat(deleted.fileSequenceNumber()).isEqualTo(SOURCE_TRACKING_ADDED.fileSequenceNumber()); assertThat(deleted.modifiedSnapshotId()).isEqualTo(SOURCE_TRACKING_ADDED.modifiedSnapshotId()); assertThat(deleted.firstRowId()).isEqualTo(SOURCE_TRACKING_ADDED.firstRowId()); + assertThat(deleted.snapshotTimestampMicros()) + .isEqualTo(SOURCE_TRACKING_ADDED.snapshotTimestampMicros()); } @Test @@ -88,13 +95,23 @@ void replaceUpdatesSnapshotIdAndPreservesRest() { assertThat(replaced.fileSequenceNumber()).isEqualTo(SOURCE_TRACKING_ADDED.fileSequenceNumber()); assertThat(replaced.modifiedSnapshotId()).isEqualTo(SOURCE_TRACKING_ADDED.modifiedSnapshotId()); assertThat(replaced.firstRowId()).isEqualTo(SOURCE_TRACKING_ADDED.firstRowId()); + assertThat(replaced.snapshotTimestampMicros()) + .isEqualTo(SOURCE_TRACKING_ADDED.snapshotTimestampMicros()); } @Test void sourceDvPositionsAreNotCarriedForward() { Tracking source = new TrackingStruct( - EntryStatus.ADDED, 42L, 10L, 10L, 43L, 1000L, new byte[] {1, 2}, new byte[] {3, 4}); + EntryStatus.ADDED, + 42L, // snapshotId + 10L, // dataSequenceNumber + 10L, // fileSequenceNumber + 43L, // modifiedSnapshotId + 1000L, // firstRowId + new byte[] {1, 2}, // deletedPositions + new byte[] {3, 4}, // replacedPositions + 1_700_000_000_123_456L); // snapshotTimestampMicros Tracking existing = TrackingBuilder.from(source, 1L).build(); assertThat(existing.deletedPositions()).isNull(); @@ -123,7 +140,7 @@ void dvUpdatedProducesModifiedAndAdvancesModifiedSnapshotId() { } @Test - void columnFilesUpdatedProducesModifiedAndResetsDataSequenceNumber() { + void columnFilesUpdatedProducesModifiedAndResetsDataSequenceNumberAndTimestamp() { Tracking modified = TrackingBuilder.from(SOURCE_TRACKING_ADDED, 999L).columnFilesUpdated().build(); @@ -132,6 +149,8 @@ void columnFilesUpdatedProducesModifiedAndResetsDataSequenceNumber() { // data sequence number is reset to inherit the new snapshot's sequence number assertThat(modified.dataSequenceNumber()).isNull(); assertThat(modified.fileSequenceNumber()).isEqualTo(SOURCE_TRACKING_ADDED.fileSequenceNumber()); + // snapshot timestamp is reset to inherit the new snapshot's timestamp + assertThat(modified.snapshotTimestampMicros()).isNull(); } @Test @@ -192,7 +211,7 @@ void sourceBuildersRejectSourceWithoutSequenceNumbers() { .hasMessage("Invalid tracking source: data sequence number is null"); TrackingStruct missingFileSeq = - new TrackingStruct(EntryStatus.ADDED, 42L, 10L, null, null, null, null, null); + new TrackingStruct(EntryStatus.ADDED, 42L, 10L, null, null, null, null, null, null); assertThatThrownBy(() -> TrackingBuilder.from(missingFileSeq, 1L)) .isInstanceOf(IllegalArgumentException.class) @@ -224,7 +243,9 @@ private static Stream terminalTransitionCases() { @MethodSource("terminalTransitionCases") void rejectsTransitionsFromTerminalStatus( EntryStatus sourceStatus, Consumer factoryCall) { - Tracking source = new TrackingStruct(sourceStatus, 42L, 10L, 10L, 43L, 1000L, null, null); + Tracking source = + new TrackingStruct( + sourceStatus, 42L, 10L, 10L, 43L, 1000L, null, null, 1_700_000_000_123_456L); assertThatThrownBy(() -> factoryCall.accept(source)) .isInstanceOf(IllegalStateException.class) @@ -275,6 +296,8 @@ void carryForwardFromModifiedSourceChangesToExisting() { assertThat(carried.fileSequenceNumber()) .isEqualTo(SOURCE_TRACKING_MODIFIED.fileSequenceNumber()); assertThat(carried.firstRowId()).isEqualTo(SOURCE_TRACKING_MODIFIED.firstRowId()); + assertThat(carried.snapshotTimestampMicros()) + .isEqualTo(SOURCE_TRACKING_MODIFIED.snapshotTimestampMicros()); } @Test diff --git a/core/src/test/java/org/apache/iceberg/TestTrackingStruct.java b/core/src/test/java/org/apache/iceberg/TestTrackingStruct.java index eea6e4e0a78d..578a3cf31945 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackingStruct.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackingStruct.java @@ -41,7 +41,15 @@ class TestTrackingStruct { void fieldAccess() { TrackingStruct tracking = new TrackingStruct( - EntryStatus.ADDED, 42L, 10L, 11L, 43L, 1000L, DELETED_POSITIONS, REPLACED_POSITIONS); + EntryStatus.ADDED, + 42L, // snapshotId + 10L, // dataSequenceNumber + 11L, // fileSequenceNumber + 43L, // modifiedSnapshotId + 1000L, // firstRowId + DELETED_POSITIONS, + REPLACED_POSITIONS, + 1_700_000_000_123_456L); // snapshotTimestampMicros tracking.set(MANIFEST_LOCATION_ORDINAL, "manifest-location"); tracking.set(MANIFEST_POSITION_ORDINAL, 7L); @@ -53,6 +61,7 @@ void fieldAccess() { assertThat(tracking.firstRowId()).isEqualTo(1000L); assertThat(tracking.deletedPositions()).isEqualTo(ByteBuffer.wrap(DELETED_POSITIONS)); assertThat(tracking.replacedPositions()).isEqualTo(ByteBuffer.wrap(REPLACED_POSITIONS)); + assertThat(tracking.snapshotTimestampMicros()).isEqualTo(1_700_000_000_123_456L); assertThat(tracking.manifestLocation()).isEqualTo("manifest-location"); assertThat(tracking.manifestPos()).isEqualTo(7L); } @@ -69,6 +78,7 @@ void setByPosition() { tracking.set(pos("first_row_id"), 1000L); tracking.set(pos("deleted_positions"), ByteBuffer.wrap(DELETED_POSITIONS)); tracking.set(pos("replaced_positions"), ByteBuffer.wrap(REPLACED_POSITIONS)); + tracking.set(pos("snapshot_timestamp"), 1_700_000_000_123_456L); tracking.set(MANIFEST_LOCATION_ORDINAL, "manifest-location"); tracking.set(MANIFEST_POSITION_ORDINAL, 7L); @@ -80,6 +90,7 @@ void setByPosition() { assertThat(tracking.firstRowId()).isEqualTo(1000L); assertThat(tracking.deletedPositions()).isEqualTo(ByteBuffer.wrap(DELETED_POSITIONS)); assertThat(tracking.replacedPositions()).isEqualTo(ByteBuffer.wrap(REPLACED_POSITIONS)); + assertThat(tracking.snapshotTimestampMicros()).isEqualTo(1_700_000_000_123_456L); assertThat(tracking.manifestLocation()).isEqualTo("manifest-location"); assertThat(tracking.manifestPos()).isEqualTo(7L); } @@ -88,7 +99,15 @@ void setByPosition() { void getByPosition() { TrackingStruct tracking = new TrackingStruct( - EntryStatus.ADDED, 42L, 10L, 11L, 43L, 1000L, DELETED_POSITIONS, REPLACED_POSITIONS); + EntryStatus.ADDED, + 42L, // snapshotId + 10L, // dataSequenceNumber + 11L, // fileSequenceNumber + 43L, // modifiedSnapshotId + 1000L, // firstRowId + DELETED_POSITIONS, + REPLACED_POSITIONS, + 1_700_000_000_123_456L); // snapshotTimestampMicros tracking.set(MANIFEST_LOCATION_ORDINAL, "manifest-location"); tracking.set(MANIFEST_POSITION_ORDINAL, 7L); @@ -102,6 +121,8 @@ void getByPosition() { .isEqualTo(ByteBuffer.wrap(DELETED_POSITIONS)); assertThat(tracking.get(pos("replaced_positions"), ByteBuffer.class)) .isEqualTo(ByteBuffer.wrap(REPLACED_POSITIONS)); + assertThat(tracking.get(pos("snapshot_timestamp"), Long.class)) + .isEqualTo(1_700_000_000_123_456L); assertThat(tracking.get(MANIFEST_LOCATION_ORDINAL, String.class)) .isEqualTo("manifest-location"); assertThat(tracking.get(MANIFEST_POSITION_ORDINAL, Long.class)).isEqualTo(7L); @@ -111,7 +132,15 @@ void getByPosition() { void copy() { TrackingStruct tracking = new TrackingStruct( - EntryStatus.MODIFIED, 42L, 10L, 11L, 43L, 1000L, DELETED_POSITIONS, REPLACED_POSITIONS); + EntryStatus.MODIFIED, + 42L, // snapshotId + 10L, // dataSequenceNumber + 11L, // fileSequenceNumber + 43L, // modifiedSnapshotId + 1000L, // firstRowId + DELETED_POSITIONS, + REPLACED_POSITIONS, + 1_700_000_000_123_456L); // snapshotTimestampMicros tracking.set(MANIFEST_LOCATION_ORDINAL, "manifest-location"); tracking.set(MANIFEST_POSITION_ORDINAL, 7L); @@ -125,6 +154,7 @@ void copy() { assertThat(copy.firstRowId()).isEqualTo(tracking.firstRowId()); assertThat(copy.deletedPositions()).isEqualTo(tracking.deletedPositions()); assertThat(copy.replacedPositions()).isEqualTo(tracking.replacedPositions()); + assertThat(copy.snapshotTimestampMicros()).isEqualTo(tracking.snapshotTimestampMicros()); assertThat(copy.manifestLocation()).isEqualTo(tracking.manifestLocation()); assertThat(copy.manifestPos()).isEqualTo(tracking.manifestPos()); @@ -136,18 +166,21 @@ void copy() { @ParameterizedTest @EnumSource(EntryStatus.class) void inheritSnapshotIdOnly(EntryStatus status) { - TrackingStruct tracking = new TrackingStruct(status, null, null, null, null, null, null, null); + TrackingStruct tracking = + new TrackingStruct(status, null, null, null, null, null, null, null, null); tracking.inherit(100L); assertThat(tracking.snapshotId()).isEqualTo(100L); assertThat(tracking.dataSequenceNumber()).isNull(); + assertThat(tracking.snapshotTimestampMicros()).isNull(); } @ParameterizedTest @EnumSource(EntryStatus.class) void inheritSnapshotId(EntryStatus status) { - TrackingStruct tracking = new TrackingStruct(status, null, null, null, null, null, null, null); + TrackingStruct tracking = + new TrackingStruct(status, null, null, null, null, null, null, null, null); tracking.inherit(100L, 60L); @@ -157,7 +190,8 @@ void inheritSnapshotId(EntryStatus status) { @ParameterizedTest @EnumSource(EntryStatus.class) void inheritancePreservesExplicitValues(EntryStatus status) { - TrackingStruct tracking = new TrackingStruct(status, 200L, 75L, 76L, null, null, null, null); + TrackingStruct tracking = + new TrackingStruct(status, 200L, 75L, 76L, null, null, null, null, null); tracking.inherit(100L, 60L); @@ -170,7 +204,8 @@ void inheritancePreservesExplicitValues(EntryStatus status) { @EnumSource(EntryStatus.class) void inheritanceAllStatusesInheritSeq0(EntryStatus status) { // Sequence number 0 is inherited for all statuses for reading pre-seq v1 tables - TrackingStruct tracking = new TrackingStruct(status, 42L, null, null, null, null, null, null); + TrackingStruct tracking = + new TrackingStruct(status, 42L, null, null, null, null, null, null, null); tracking.inherit(100L, 0L); @@ -181,7 +216,7 @@ void inheritanceAllStatusesInheritSeq0(EntryStatus status) { @Test void inheritanceAddedEntriesInheritSequenceNumber() { TrackingStruct tracking = - new TrackingStruct(EntryStatus.ADDED, 42L, null, null, null, null, null, null); + new TrackingStruct(EntryStatus.ADDED, 42L, null, null, null, null, null, null, null); tracking.inherit(100L, 60L); @@ -192,7 +227,7 @@ void inheritanceAddedEntriesInheritSequenceNumber() { @Test void inheritanceModifiedEntriesInheritDataSequenceNumber() { TrackingStruct tracking = - new TrackingStruct(EntryStatus.MODIFIED, 42L, null, null, null, null, null, null); + new TrackingStruct(EntryStatus.MODIFIED, 42L, null, null, null, null, null, null, null); tracking.inherit(100L, 60L); @@ -206,7 +241,8 @@ void inheritanceModifiedEntriesInheritDataSequenceNumber() { @ParameterizedTest @FieldSource("NON_INHERITING_STATUSES") void inheritanceWithNonInheritingStatus(EntryStatus status) { - TrackingStruct tracking = new TrackingStruct(status, 42L, null, null, null, null, null, null); + TrackingStruct tracking = + new TrackingStruct(status, 42L, null, null, null, null, null, null, null); tracking.inherit(100L, 60L); @@ -222,7 +258,8 @@ void inheritanceWithNonInheritingStatus(EntryStatus status) { @ParameterizedTest @FieldSource("ASSIGNING_STATUSES") void assignmentFirstRowIdAssigned(EntryStatus status) { - TrackingStruct tracking = new TrackingStruct(status, 42L, null, null, null, null, null, null); + TrackingStruct tracking = + new TrackingStruct(status, 42L, null, null, null, null, null, null, null); assertThat(tracking.assignFirstRowId(10_000L)).as("Should return true when assigned").isTrue(); @@ -232,7 +269,8 @@ void assignmentFirstRowIdAssigned(EntryStatus status) { @ParameterizedTest @FieldSource("NON_ASSIGNING_STATUSES") void assignmentFirstRowIdNotAssigned(EntryStatus status) { - TrackingStruct tracking = new TrackingStruct(status, 42L, null, null, null, null, null, null); + TrackingStruct tracking = + new TrackingStruct(status, 42L, null, null, null, null, null, null, null); assertThat(tracking.assignFirstRowId(10_000L)) .as("Should return false when not assigned") @@ -244,7 +282,8 @@ void assignmentFirstRowIdNotAssigned(EntryStatus status) { @ParameterizedTest @EnumSource(EntryStatus.class) void assignmentFirstRowIdAlreadyAssigned(EntryStatus status) { - TrackingStruct tracking = new TrackingStruct(status, 42L, null, null, null, 5_000L, null, null); + TrackingStruct tracking = + new TrackingStruct(status, 42L, null, null, null, 5_000L, null, null, null); assertThat(tracking.assignFirstRowId(10_000L)) .as("Should return false when not assigned") @@ -259,7 +298,7 @@ void assignmentFirstRowIdAlreadyAssigned(EntryStatus status) { @EnumSource(EntryStatus.class) void assignmentNullFirstRowId(EntryStatus status) { TrackingStruct tracking = - new TrackingStruct(status, 42L, null, null, null, 10_000L, null, null); + new TrackingStruct(status, 42L, null, null, null, 10_000L, null, null, null); assertThat(tracking.assignFirstRowId(null)) .as("Should return false when not assigned") @@ -271,7 +310,8 @@ void assignmentNullFirstRowId(EntryStatus status) { @ParameterizedTest @EnumSource(EntryStatus.class) void isLiveDelegatesToStatus(EntryStatus status) { - TrackingStruct tracking = new TrackingStruct(status, null, null, null, null, null, null, null); + TrackingStruct tracking = + new TrackingStruct(status, null, null, null, null, null, null, null, null); assertThat(tracking.isLive()).isEqualTo(status.isLive()); } @@ -280,7 +320,15 @@ void isLiveDelegatesToStatus(EntryStatus status) { void internalSetIgnoresUnknownOrdinal() { TrackingStruct tracking = new TrackingStruct( - EntryStatus.ADDED, 42L, 10L, 11L, 43L, 1000L, DELETED_POSITIONS, REPLACED_POSITIONS); + EntryStatus.ADDED, + 42L, // snapshotId + 10L, // dataSequenceNumber + 11L, // fileSequenceNumber + 43L, // modifiedSnapshotId + 1000L, // firstRowId + DELETED_POSITIONS, + REPLACED_POSITIONS, + 1_700_000_000_123_456L); // snapshotTimestampMicros // unknown ordinals from a newer format version are silently ignored tracking.internalSet(99, "value from a newer format"); @@ -294,6 +342,7 @@ void internalSetIgnoresUnknownOrdinal() { assertThat(tracking.firstRowId()).isEqualTo(1000L); assertThat(tracking.deletedPositions()).isEqualTo(ByteBuffer.wrap(DELETED_POSITIONS)); assertThat(tracking.replacedPositions()).isEqualTo(ByteBuffer.wrap(REPLACED_POSITIONS)); + assertThat(tracking.snapshotTimestampMicros()).isEqualTo(1_700_000_000_123_456L); } @Test @@ -321,7 +370,15 @@ void serializationRoundTrip(TestHelpers.RoundTripSerializer roundTripS throws IOException, ClassNotFoundException { TrackingStruct tracking = new TrackingStruct( - EntryStatus.MODIFIED, 42L, 10L, 11L, 43L, 1000L, DELETED_POSITIONS, REPLACED_POSITIONS); + EntryStatus.MODIFIED, + 42L, // snapshotId + 10L, // dataSequenceNumber + 11L, // fileSequenceNumber + 43L, // modifiedSnapshotId + 1000L, // firstRowId + DELETED_POSITIONS, + REPLACED_POSITIONS, + 1_700_000_000_123_456L); // snapshotTimestampMicros tracking.set(MANIFEST_LOCATION_ORDINAL, "manifest-location"); tracking.set(MANIFEST_POSITION_ORDINAL, 7L); @@ -335,6 +392,7 @@ void serializationRoundTrip(TestHelpers.RoundTripSerializer roundTripS assertThat(deserialized.firstRowId()).isEqualTo(1000L); assertThat(deserialized.deletedPositions()).isEqualTo(ByteBuffer.wrap(DELETED_POSITIONS)); assertThat(deserialized.replacedPositions()).isEqualTo(ByteBuffer.wrap(REPLACED_POSITIONS)); + assertThat(deserialized.snapshotTimestampMicros()).isEqualTo(1_700_000_000_123_456L); assertThat(deserialized.manifestLocation()).isEqualTo("manifest-location"); assertThat(deserialized.manifestPos()).isEqualTo(7L); } diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index e7e44b3d1077..3dab3010921b 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -83,6 +83,7 @@ class TestV4ManifestReader { } private static final long MANIFEST_SEQ = 309L; + private static final long SNAPSHOT_TS = 1_700_000_000_123_456L; private static final DeletionVector DV = deletionVector("s3://bucket/dv.puffin"); private static final ManifestInfo MANIFEST_INFO = @@ -317,19 +318,24 @@ public void statusFilter(FileFormat format) throws IOException { List files = List.of( dataFile( - new TrackingStruct(EntryStatus.ADDED, 42L, null, null, null, null, null, null), + new TrackingStruct( + EntryStatus.ADDED, 42L, null, null, null, null, null, null, null), "s3://bucket/added.parquet"), dataFile( - new TrackingStruct(EntryStatus.MODIFIED, 40L, 5L, 5L, 42L, 5_000L, null, null), + new TrackingStruct( + EntryStatus.MODIFIED, 40L, 5L, 5L, 42L, 5_000L, null, null, SNAPSHOT_TS), "s3://bucket/modified.parquet"), dataFile( - new TrackingStruct(EntryStatus.DELETED, 42L, 2L, 2L, null, 1_000L, null, null), + new TrackingStruct( + EntryStatus.DELETED, 42L, 2L, 2L, null, 1_000L, null, null, SNAPSHOT_TS), "s3://bucket/deleted.parquet"), dataFile( - new TrackingStruct(EntryStatus.EXISTING, 38L, 4L, 4L, null, 3_000L, null, null), + new TrackingStruct( + EntryStatus.EXISTING, 38L, 4L, 4L, null, 3_000L, null, null, SNAPSHOT_TS), "s3://bucket/existing.parquet"), dataFile( - new TrackingStruct(EntryStatus.REPLACED, 42L, 2L, 2L, 40L, 2_000L, null, null), + new TrackingStruct( + EntryStatus.REPLACED, 42L, 2L, 2L, 40L, 2_000L, null, null, SNAPSHOT_TS), "s3://bucket/replaced.parquet")); ManifestFile manifest = writeManifest(format, UNPARTITIONED_TYPE, files); @@ -340,19 +346,31 @@ public void statusFilter(FileFormat format) throws IOException { // inherits seq number and assigned first row ID dataFile( new TrackingStruct( - EntryStatus.ADDED, 42L, MANIFEST_SEQ, MANIFEST_SEQ, null, 10_000L, null, null), + EntryStatus.ADDED, + 42L, // snapshotId + MANIFEST_SEQ, + MANIFEST_SEQ, + null, // modifiedSnapshotId + 10_000L, // firstRowId + null, // deletedPositions + null, // replacedPositions + null), // snapshotTimestampMicros "s3://bucket/added.parquet"), dataFile( - new TrackingStruct(EntryStatus.MODIFIED, 40L, 5L, 5L, 42L, 5_000L, null, null), + new TrackingStruct( + EntryStatus.MODIFIED, 40L, 5L, 5L, 42L, 5_000L, null, null, SNAPSHOT_TS), "s3://bucket/modified.parquet"), dataFile( - new TrackingStruct(EntryStatus.DELETED, 42L, 2L, 2L, null, 1_000L, null, null), + new TrackingStruct( + EntryStatus.DELETED, 42L, 2L, 2L, null, 1_000L, null, null, SNAPSHOT_TS), "s3://bucket/deleted.parquet"), dataFile( - new TrackingStruct(EntryStatus.EXISTING, 38L, 4L, 4L, null, 3_000L, null, null), + new TrackingStruct( + EntryStatus.EXISTING, 38L, 4L, 4L, null, 3_000L, null, null, SNAPSHOT_TS), "s3://bucket/existing.parquet"), dataFile( - new TrackingStruct(EntryStatus.REPLACED, 42L, 2L, 2L, 40L, 2_000L, null, null), + new TrackingStruct( + EntryStatus.REPLACED, 42L, 2L, 2L, 40L, 2_000L, null, null, SNAPSHOT_TS), "s3://bucket/replaced.parquet")); V4ManifestReader.Builder builder = @@ -371,19 +389,24 @@ public void mdvFilter(FileFormat format) throws IOException { List files = List.of( dataFile( - new TrackingStruct(EntryStatus.ADDED, 42L, null, null, null, null, null, null), + new TrackingStruct( + EntryStatus.ADDED, 42L, null, null, null, null, null, null, null), "s3://bucket/added.parquet"), dataFile( - new TrackingStruct(EntryStatus.MODIFIED, 40L, 5L, 5L, 42L, 5_000L, null, null), + new TrackingStruct( + EntryStatus.MODIFIED, 40L, 5L, 5L, 42L, 5_000L, null, null, SNAPSHOT_TS), "s3://bucket/modified.parquet"), dataFile( - new TrackingStruct(EntryStatus.DELETED, 42L, 2L, 2L, null, 1_000L, null, null), + new TrackingStruct( + EntryStatus.DELETED, 42L, 2L, 2L, null, 1_000L, null, null, SNAPSHOT_TS), "s3://bucket/deleted.parquet"), dataFile( - new TrackingStruct(EntryStatus.EXISTING, 38L, 4L, 4L, null, 3_000L, null, null), + new TrackingStruct( + EntryStatus.EXISTING, 38L, 4L, 4L, null, 3_000L, null, null, SNAPSHOT_TS), "s3://bucket/existing.parquet"), dataFile( - new TrackingStruct(EntryStatus.REPLACED, 42L, 2L, 2L, 40L, 2_000L, null, null), + new TrackingStruct( + EntryStatus.REPLACED, 42L, 2L, 2L, 40L, 2_000L, null, null, SNAPSHOT_TS), "s3://bucket/replaced.parquet")); ManifestFile manifest = writeManifest(format, UNPARTITIONED_TYPE, files); @@ -398,21 +421,33 @@ public void mdvFilter(FileFormat format) throws IOException { // inherits seq number and assigned first row ID dataFile( new TrackingStruct( - EntryStatus.ADDED, 42L, MANIFEST_SEQ, MANIFEST_SEQ, null, 10_000L, null, null), + EntryStatus.ADDED, + 42L, // snapshotId + MANIFEST_SEQ, + MANIFEST_SEQ, + null, // modifiedSnapshotId + 10_000L, // firstRowId + null, // deletedPositions + null, // replacedPositions + null), // snapshotTimestampMicros "s3://bucket/added.parquet"), // status changed to DELETED, delete snapshot ID is unknown dataFile( - new TrackingStruct(EntryStatus.DELETED, null, 5L, 5L, 42L, 5_000L, null, null), + new TrackingStruct( + EntryStatus.DELETED, null, 5L, 5L, 42L, 5_000L, null, null, null), "s3://bucket/modified.parquet"), // status already DELETED, no modification dataFile( - new TrackingStruct(EntryStatus.DELETED, 42L, 2L, 2L, null, 1_000L, null, null), + new TrackingStruct( + EntryStatus.DELETED, 42L, 2L, 2L, null, 1_000L, null, null, SNAPSHOT_TS), "s3://bucket/deleted.parquet"), dataFile( - new TrackingStruct(EntryStatus.EXISTING, 38L, 4L, 4L, null, 3_000L, null, null), + new TrackingStruct( + EntryStatus.EXISTING, 38L, 4L, 4L, null, 3_000L, null, null, SNAPSHOT_TS), "s3://bucket/existing.parquet"), dataFile( - new TrackingStruct(EntryStatus.REPLACED, 42L, 2L, 2L, 40L, 2_000L, null, null), + new TrackingStruct( + EntryStatus.REPLACED, 42L, 2L, 2L, 40L, 2_000L, null, null, SNAPSHOT_TS), "s3://bucket/replaced.parquet")); V4ManifestReader.Builder builder = @@ -430,11 +465,12 @@ public void mdvFilter(FileFormat format) throws IOException { public void inheritanceUncommittedOnlyInheritsSnapshotId(FileFormat format) throws IOException { TrackedFile withSnapshotId = dataFile( - new TrackingStruct(EntryStatus.ADDED, 1234567L, null, null, null, null, null, null), + new TrackingStruct( + EntryStatus.ADDED, 1234567L, null, null, null, null, null, null, null), "s3://bucket/table/file-b.parquet"); TrackedFile withoutSnapshotId = dataFile( - new TrackingStruct(EntryStatus.ADDED, null, null, null, null, null, null, null), + new TrackingStruct(EntryStatus.ADDED, null, null, null, null, null, null, null, null), "s3://bucket/table/file-a.parquet"); ManifestFile manifest = @@ -458,11 +494,13 @@ public void inheritanceUncommittedOnlyInheritsSnapshotId(FileFormat format) thro List expected = List.of( dataFile( - new TrackingStruct(EntryStatus.ADDED, 1234567L, null, null, null, null, null, null), + new TrackingStruct( + EntryStatus.ADDED, 1234567L, null, null, null, null, null, null, null), "s3://bucket/table/file-b.parquet"), // inherits only snapshot ID dataFile( - new TrackingStruct(EntryStatus.ADDED, 34L, null, null, null, null, null, null), + new TrackingStruct( + EntryStatus.ADDED, 34L, null, null, null, null, null, null, null), "s3://bucket/table/file-a.parquet")); assertThat(actual) @@ -475,11 +513,13 @@ public void inheritanceUncommittedOnlyInheritsSnapshotId(FileFormat format) thro public void inheritanceSnapshotId(FileFormat format) throws IOException { TrackedFile withSnapshotId = dataFile( - new TrackingStruct(EntryStatus.ADDED, 1234567L, 5L, 5L, null, 5_000L, null, null), + new TrackingStruct( + EntryStatus.ADDED, 1234567L, 5L, 5L, null, 5_000L, null, null, SNAPSHOT_TS), "s3://bucket/table/file-b.parquet"); TrackedFile withoutSnapshotId = dataFile( - new TrackingStruct(EntryStatus.ADDED, null, 5L, 5L, null, 5_100L, null, null), + new TrackingStruct( + EntryStatus.ADDED, null, 5L, 5L, null, 5_100L, null, null, SNAPSHOT_TS), "s3://bucket/table/file-a.parquet"); ManifestFile manifest = @@ -497,11 +537,13 @@ public void inheritanceSnapshotId(FileFormat format) throws IOException { List expected = List.of( dataFile( - new TrackingStruct(EntryStatus.ADDED, 1234567L, 5L, 5L, null, 5_000L, null, null), + new TrackingStruct( + EntryStatus.ADDED, 1234567L, 5L, 5L, null, 5_000L, null, null, SNAPSHOT_TS), "s3://bucket/table/file-b.parquet"), // inherits only snapshot ID dataFile( - new TrackingStruct(EntryStatus.ADDED, 34L, 5L, 5L, null, 5_100L, null, null), + new TrackingStruct( + EntryStatus.ADDED, 34L, 5L, 5L, null, 5_100L, null, null, SNAPSHOT_TS), "s3://bucket/table/file-a.parquet")); assertThat(actual) @@ -515,11 +557,12 @@ public void inheritanceModifiedSnapshotIdNotInherited(FileFormat format) throws TrackedFile withModifiedSnapshotId = dataFile( new TrackingStruct( - EntryStatus.ADDED, SNAPSHOT_ID, 5L, 5L, 1234567L, 5_000L, null, null), + EntryStatus.ADDED, SNAPSHOT_ID, 5L, 5L, 1234567L, 5_000L, null, null, SNAPSHOT_TS), "s3://bucket/table/file-b.parquet"); TrackedFile withoutModifiedSnapshotId = dataFile( - new TrackingStruct(EntryStatus.ADDED, SNAPSHOT_ID, 5L, 5L, null, 5_100L, null, null), + new TrackingStruct( + EntryStatus.ADDED, SNAPSHOT_ID, 5L, 5L, null, 5_100L, null, null, SNAPSHOT_TS), "s3://bucket/table/file-a.parquet"); ManifestFile manifest = @@ -540,12 +583,20 @@ public void inheritanceModifiedSnapshotIdNotInherited(FileFormat format) throws List.of( dataFile( new TrackingStruct( - EntryStatus.ADDED, SNAPSHOT_ID, 5L, 5L, 1234567L, 5_000L, null, null), + EntryStatus.ADDED, + SNAPSHOT_ID, + 5L, + 5L, + 1234567L, + 5_000L, + null, + null, + SNAPSHOT_TS), "s3://bucket/table/file-b.parquet"), // inherits only snapshot ID dataFile( new TrackingStruct( - EntryStatus.ADDED, SNAPSHOT_ID, 5L, 5L, null, 5_100L, null, null), + EntryStatus.ADDED, SNAPSHOT_ID, 5L, 5L, null, 5_100L, null, null, SNAPSHOT_TS), "s3://bucket/table/file-a.parquet")); assertThat(actual) @@ -559,12 +610,12 @@ public void inheritanceAddedSequenceNumbers(FileFormat format) throws IOExceptio TrackedFile withoutSeq = dataFile( new TrackingStruct( - EntryStatus.ADDED, SNAPSHOT_ID, null, null, null, 5_000L, null, null), + EntryStatus.ADDED, SNAPSHOT_ID, null, null, null, 5_000L, null, null, null), "s3://bucket/table/file-a.parquet"); TrackedFile withDataSeq = dataFile( new TrackingStruct( - EntryStatus.ADDED, SNAPSHOT_ID, 500L, 500L, null, 5_100L, null, null), + EntryStatus.ADDED, SNAPSHOT_ID, 500L, 500L, null, 5_100L, null, null, SNAPSHOT_TS), "s3://bucket/table/file-b.parquet"); ManifestFile manifest = @@ -584,12 +635,20 @@ public void inheritanceAddedSequenceNumbers(FileFormat format) throws IOExceptio // inherits sequence numbers dataFile( new TrackingStruct( - EntryStatus.ADDED, SNAPSHOT_ID, 5L, 5L, null, 5_000L, null, null), + EntryStatus.ADDED, SNAPSHOT_ID, 5L, 5L, null, 5_000L, null, null, null), "s3://bucket/table/file-a.parquet"), // does not inherit sequence numbers dataFile( new TrackingStruct( - EntryStatus.ADDED, SNAPSHOT_ID, 500L, 500L, null, 5_100L, null, null), + EntryStatus.ADDED, + SNAPSHOT_ID, + 500L, + 500L, + null, + 5_100L, + null, + null, + SNAPSHOT_TS), "s3://bucket/table/file-b.parquet")); assertThat(actual) @@ -603,7 +662,7 @@ public void inheritanceModifiedDataSequenceNumber(FileFormat format) throws IOEx TrackedFile withoutDataSeq = dataFileWithColumnFiles( new TrackingStruct( - EntryStatus.MODIFIED, SNAPSHOT_ID, null, 3L, 34L, 5_000L, null, null), + EntryStatus.MODIFIED, SNAPSHOT_ID, null, 3L, 34L, 5_000L, null, null, null), "s3://bucket/table/file-a.parquet", COLUMN_FILES); @@ -624,7 +683,7 @@ public void inheritanceModifiedDataSequenceNumber(FileFormat format) throws IOEx List.of( dataFileWithColumnFiles( new TrackingStruct( - EntryStatus.MODIFIED, SNAPSHOT_ID, 5L, 3L, 34L, 5_000L, null, null), + EntryStatus.MODIFIED, SNAPSHOT_ID, 5L, 3L, 34L, 5_000L, null, null, null), "s3://bucket/table/file-a.parquet", COLUMN_FILES)); @@ -638,11 +697,13 @@ public void inheritanceModifiedDataSequenceNumber(FileFormat format) throws IOEx public void inheritanceFirstRowId(FileFormat format) throws IOException { TrackedFile withoutFirstRowId = dataFile( - new TrackingStruct(EntryStatus.ADDED, SNAPSHOT_ID, 5L, 5L, null, null, null, null), + new TrackingStruct( + EntryStatus.ADDED, SNAPSHOT_ID, 5L, 5L, null, null, null, null, SNAPSHOT_TS), "s3://bucket/table/file-a.parquet"); TrackedFile withFirstRowId = dataFile( - new TrackingStruct(EntryStatus.EXISTING, SNAPSHOT_ID, 5L, 5L, null, 5_000L, null, null), + new TrackingStruct( + EntryStatus.EXISTING, SNAPSHOT_ID, 5L, 5L, null, 5_000L, null, null, SNAPSHOT_TS), "s3://bucket/table/file-c.parquet"); ManifestFile manifest = @@ -661,12 +722,20 @@ public void inheritanceFirstRowId(FileFormat format) throws IOException { // assigns a new first row ID dataFile( new TrackingStruct( - EntryStatus.ADDED, SNAPSHOT_ID, 5L, 5L, null, 10_000L, null, null), + EntryStatus.ADDED, SNAPSHOT_ID, 5L, 5L, null, 10_000L, null, null, SNAPSHOT_TS), "s3://bucket/table/file-a.parquet"), // does not modify first row ID dataFile( new TrackingStruct( - EntryStatus.EXISTING, SNAPSHOT_ID, 5L, 5L, null, 5_000L, null, null), + EntryStatus.EXISTING, + SNAPSHOT_ID, + 5L, + 5L, + null, + 5_000L, + null, + null, + SNAPSHOT_TS), "s3://bucket/table/file-c.parquet")); assertThat(actual) @@ -678,9 +747,11 @@ public void inheritanceFirstRowId(FileFormat format) throws IOException { @FieldSource("MANIFEST_FORMATS") public void inheritanceFirstRowIdIncrementsByRecordCount(FileFormat format) throws IOException { Tracking trackingWithoutFirstRowId = - new TrackingStruct(EntryStatus.ADDED, SNAPSHOT_ID, null, null, null, null, null, null); + new TrackingStruct( + EntryStatus.ADDED, SNAPSHOT_ID, null, null, null, null, null, null, null); Tracking trackingWithFirstRowId = - new TrackingStruct(EntryStatus.EXISTING, SNAPSHOT_ID, 2L, 2L, null, 5_000L, null, null); + new TrackingStruct( + EntryStatus.EXISTING, SNAPSHOT_ID, 2L, 2L, null, 5_000L, null, null, SNAPSHOT_TS); TrackedFile fileA = dataFile(trackingWithoutFirstRowId, "s3://bucket/table/file-a.parquet"); TrackedFile fileB = dataFile(trackingWithoutFirstRowId, "s3://bucket/table/file-b.parquet"); @@ -705,23 +776,31 @@ public void inheritanceFirstRowIdIncrementsByRecordCount(FileFormat format) thro List.of( dataFile( new TrackingStruct( - EntryStatus.ADDED, SNAPSHOT_ID, 5L, 5L, null, 10_000L, null, null), + EntryStatus.ADDED, SNAPSHOT_ID, 5L, 5L, null, 10_000L, null, null, null), "s3://bucket/table/file-a.parquet"), dataFile( new TrackingStruct( - EntryStatus.ADDED, SNAPSHOT_ID, 5L, 5L, null, 10_100L, null, null), + EntryStatus.ADDED, SNAPSHOT_ID, 5L, 5L, null, 10_100L, null, null, null), "s3://bucket/table/file-b.parquet"), dataFile( new TrackingStruct( - EntryStatus.ADDED, SNAPSHOT_ID, 5L, 5L, null, 10_200L, null, null), + EntryStatus.ADDED, SNAPSHOT_ID, 5L, 5L, null, 10_200L, null, null, null), "s3://bucket/table/file-c.parquet"), dataFile( new TrackingStruct( - EntryStatus.EXISTING, SNAPSHOT_ID, 2L, 2L, null, 5_000L, null, null), + EntryStatus.EXISTING, + SNAPSHOT_ID, + 2L, + 2L, + null, + 5_000L, + null, + null, + SNAPSHOT_TS), "s3://bucket/table/file-d.parquet"), dataFile( new TrackingStruct( - EntryStatus.ADDED, SNAPSHOT_ID, 5L, 5L, null, 10_300L, null, null), + EntryStatus.ADDED, SNAPSHOT_ID, 5L, 5L, null, 10_300L, null, null, null), "s3://bucket/table/file-e.parquet")); assertThat(actual) @@ -759,11 +838,13 @@ void inheritanceFirstRowIdIsIndependentBetweenIterators(FileFormat format) throw @FieldSource("MANIFEST_FORMATS") public void inheritanceUncommittedSkipsFirstRowId(FileFormat format) throws IOException { Tracking trackingWithoutFirstRowId = - new TrackingStruct(EntryStatus.ADDED, SNAPSHOT_ID, null, null, null, null, null, null); + new TrackingStruct( + EntryStatus.ADDED, SNAPSHOT_ID, null, null, null, null, null, null, null); TrackedFile withoutFirstRowId = dataFile(trackingWithoutFirstRowId, "s3://bucket/table/file-a.parquet"); Tracking trackingWithFirstRowId = - new TrackingStruct(EntryStatus.EXISTING, SNAPSHOT_ID, null, null, null, 5_000L, null, null); + new TrackingStruct( + EntryStatus.EXISTING, SNAPSHOT_ID, null, null, null, 5_000L, null, null, null); TrackedFile withFirstRowId = dataFile(trackingWithFirstRowId, "s3://bucket/table/file-c.parquet"); @@ -786,11 +867,13 @@ public void inheritanceUncommittedSkipsFirstRowId(FileFormat format) throws IOEx @FieldSource("MANIFEST_FORMATS") public void inheritanceFirstRowIdNullFromUpgradedTable(FileFormat format) throws IOException { Tracking trackingWithoutFirstRowId = - new TrackingStruct(EntryStatus.ADDED, SNAPSHOT_ID, null, null, null, null, null, null); + new TrackingStruct( + EntryStatus.ADDED, SNAPSHOT_ID, null, null, null, null, null, null, null); TrackedFile withoutFirstRowId = dataFile(trackingWithoutFirstRowId, "s3://bucket/table/file-a.parquet"); Tracking trackingWithFirstRowId = - new TrackingStruct(EntryStatus.EXISTING, SNAPSHOT_ID, null, null, null, 5_000L, null, null); + new TrackingStruct( + EntryStatus.EXISTING, SNAPSHOT_ID, null, null, null, 5_000L, null, null, null); TrackedFile withFirstRowId = dataFile(trackingWithFirstRowId, "s3://bucket/table/file-c.parquet"); @@ -812,7 +895,8 @@ public void inheritanceFirstRowIdNullFromUpgradedTable(FileFormat format) throws @FieldSource("MANIFEST_FORMATS") public void inheritanceFirstRowIdSkipsManifests(FileFormat format) throws IOException { Tracking trackingWithoutFirstRowId = - new TrackingStruct(EntryStatus.ADDED, SNAPSHOT_ID, null, null, null, null, null, null); + new TrackingStruct( + EntryStatus.ADDED, SNAPSHOT_ID, null, null, null, null, null, null, null); TrackedFile dataFile = dataFile(trackingWithoutFirstRowId, "s3://bucket/table/file-a.parquet"); TrackedFile dataManifest = V4TestHelpers.manifestRefWithTracking( @@ -2035,7 +2119,8 @@ private static TrackedFile unpartitionedFileWithoutStats(String location) { null, // modified snapshot id null, // first row id null, // deleted positions - null); // replaced positions + null, // replaced positions + SNAPSHOT_TS); // snapshot timestamp micros return dataFile(tracking, location); } diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestWriter.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestWriter.java index 43dfdb39ce18..f9da39c05c76 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestWriter.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestWriter.java @@ -43,6 +43,8 @@ class TestV4ManifestWriter { private static final long COMMIT_SEQUENCE_NUMBER = 50L; private static final long COMMIT_FIRST_ROW_ID = 1_000L; private static final long RECORD_COUNT = 100L; + private static final long SNAPSHOT_TS = 1_700_000_000_123_456L; + private static final long COMMIT_SNAPSHOT_TS = 1_800_000_000_123_456L; private static final Schema SCHEMA = new Schema( 1, @@ -87,7 +89,8 @@ class TestV4ManifestWriter { null, // not modified COMMIT_FIRST_ROW_ID, null, // no change bitmaps - null); + null, + COMMIT_SNAPSHOT_TS); private static final DeletionVector DV_A = DeletionVectorStruct.builder() @@ -107,7 +110,8 @@ class TestV4ManifestWriter { private static final TrackedFile FILE_A = new TrackedFileStruct( - new TrackingStruct(EntryStatus.EXISTING, 40L, 35L, 34L, null, 500L, null, null), + new TrackingStruct( + EntryStatus.EXISTING, 40L, 35L, 34L, null, 500L, null, null, SNAPSHOT_TS), FileContent.DATA, "s3://bucket/db/table/data/file-a.parquet", FileFormat.PARQUET, @@ -126,7 +130,8 @@ class TestV4ManifestWriter { private static final TrackedFile FILE_A_WITH_DV = new TrackedFileStruct( - new TrackingStruct(EntryStatus.EXISTING, 40L, 35L, 34L, 41L, 500L, null, null), + new TrackingStruct( + EntryStatus.EXISTING, 40L, 35L, 34L, 41L, 500L, null, null, SNAPSHOT_TS), FileContent.DATA, "s3://bucket/db/table/data/file-a.parquet", FileFormat.PARQUET, @@ -145,7 +150,8 @@ class TestV4ManifestWriter { private static final TrackedFile FILE_B = new TrackedFileStruct( - new TrackingStruct(EntryStatus.EXISTING, 41L, 36L, 36L, null, 600L, null, null), + new TrackingStruct( + EntryStatus.EXISTING, 41L, 36L, 36L, null, 600L, null, null, SNAPSHOT_TS), FileContent.DATA, "s3://bucket/db/table/data/file-b.parquet", FileFormat.PARQUET, @@ -181,7 +187,8 @@ class TestV4ManifestWriter { private static final TrackedFile LEAF_MANIFEST = new TrackedFileStruct( - new TrackingStruct(EntryStatus.EXISTING, 40L, 35L, 35L, null, 500L, null, null), + new TrackingStruct( + EntryStatus.EXISTING, 40L, 35L, 35L, null, 500L, null, null, SNAPSHOT_TS), FileContent.DATA_MANIFEST, "s3://bucket/db/table/metadata/leaf-40-1.parquet", FileFormat.PARQUET, @@ -200,7 +207,8 @@ class TestV4ManifestWriter { private static final TrackedFile LEAF_MANIFEST_WITH_MDV = new TrackedFileStruct( - new TrackingStruct(EntryStatus.EXISTING, 40L, 35L, 35L, 41L, 500L, null, null), + new TrackingStruct( + EntryStatus.EXISTING, 40L, 35L, 35L, 41L, 500L, null, null, SNAPSHOT_TS), FileContent.DATA_MANIFEST, "s3://bucket/db/table/metadata/leaf-40-1.parquet", FileFormat.PARQUET, @@ -219,7 +227,8 @@ class TestV4ManifestWriter { private static final TrackedFile DELETE_MANIFEST = new TrackedFileStruct( - new TrackingStruct(EntryStatus.EXISTING, 40L, 35L, 35L, 41L, null, null, null), + new TrackingStruct( + EntryStatus.EXISTING, 40L, 35L, 35L, 41L, null, null, null, SNAPSHOT_TS), FileContent.DELETE_MANIFEST, "s3://bucket/db/table/metadata/leaf-40-1-deletes.parquet", FileFormat.PARQUET, @@ -272,7 +281,8 @@ public void rootDataFileAppend() throws IOException { null, // not modified COMMIT_FIRST_ROW_ID, null, // no change bitmaps - null)); + null, + null)); // snapshot timestamp } } @@ -312,7 +322,8 @@ public void rootDataFileAppendAtDataSeq() throws IOException { null, // not modified COMMIT_FIRST_ROW_ID, null, // no change bitmaps - null)); + null, + null)); // snapshot timestamp } } @@ -352,7 +363,8 @@ public void rootDataFileWithDVAppend() throws IOException { null, // added, not modified COMMIT_FIRST_ROW_ID, null, // no change bitmaps - null)); + null, + null)); // snapshot timestamp } } @@ -393,7 +405,8 @@ public void rootDataFileWithDVModify() throws IOException { COMMIT_SNAPSHOT_ID, originalTracking.firstRowId(), null, // no change bitmaps - null)); + null, + SNAPSHOT_TS)); } } @@ -434,7 +447,8 @@ public void rootDataFileExisting() throws IOException { originalTracking.modifiedSnapshotId(), originalTracking.firstRowId(), null, // no change bitmaps - null)); + null, + SNAPSHOT_TS)); } } @@ -476,7 +490,8 @@ public void rootDataFileDelete() throws IOException { originalTracking.modifiedSnapshotId(), originalTracking.firstRowId(), null, // no change bitmaps - null)); + null, + SNAPSHOT_TS)); } @Test @@ -517,7 +532,8 @@ public void rootDataFileReplace() throws IOException { originalTracking.modifiedSnapshotId(), originalTracking.firstRowId(), null, // no change bitmaps - null)); + null, + SNAPSHOT_TS)); } @Test @@ -562,7 +578,8 @@ public void rootMixedPartitionDataFiles() throws IOException { null, // not modified COMMIT_FIRST_ROW_ID, null, // no change bitmaps - null)); + null, + null)); // snapshot timestamp } @Test @@ -623,7 +640,8 @@ public void rootDataFileFirstRowIdAssignment() throws IOException { null, // not modified COMMIT_FIRST_ROW_ID, null, // no change bitmaps - null)); + null, + null)); // snapshot timestamp assertThat(files.get(1).tracking()) .usingComparator(V4TestComparators.TRACKING) .isEqualTo( @@ -635,7 +653,8 @@ public void rootDataFileFirstRowIdAssignment() throws IOException { null, // not modified COMMIT_FIRST_ROW_ID + FILE_A.recordCount(), null, // no change bitmaps - null)); + null, + null)); // snapshot timestamp } } @@ -678,7 +697,8 @@ public void rootManifestFileAppend() throws IOException { null, // not modified COMMIT_FIRST_ROW_ID, null, // no change bitmaps - null)); + null, + null)); // snapshot timestamp } } @@ -721,7 +741,8 @@ public void rootManifestFileWithMDVAppend() throws IOException { null, // not modified COMMIT_FIRST_ROW_ID, null, // no change bitmaps - null)); + null, + null)); // snapshot timestamp } } @@ -762,7 +783,8 @@ public void rootManifestFileExisting() throws IOException { originalTracking.modifiedSnapshotId(), originalTracking.firstRowId(), null, // no change bitmaps - null)); + null, + SNAPSHOT_TS)); } } @@ -804,7 +826,8 @@ public void rootManifestFileDelete() throws IOException { originalTracking.modifiedSnapshotId(), originalTracking.firstRowId(), null, // no change bitmaps - null)); + null, + SNAPSHOT_TS)); } @Test @@ -845,7 +868,8 @@ public void rootManifestFileReplace() throws IOException { originalTracking.modifiedSnapshotId(), originalTracking.firstRowId(), null, // no change bitmaps - null)); + null, + SNAPSHOT_TS)); } @Test @@ -885,7 +909,8 @@ public void rootDeleteManifestExisting() throws IOException { originalTracking.modifiedSnapshotId(), null, null, // no change bitmaps - null)); + null, + SNAPSHOT_TS)); } } @@ -927,7 +952,8 @@ public void rootDeleteManifestDelete() throws IOException { originalTracking.modifiedSnapshotId(), null, null, // no change bitmaps - null)); + null, + SNAPSHOT_TS)); } @Test @@ -1099,7 +1125,8 @@ public void rootMixedDataAndManifestFiles() throws IOException { null, // not modified COMMIT_FIRST_ROW_ID, null, // no change bitmaps - null)); + null, + null)); // snapshot timestamp assertThat(files.get(3).tracking()) .usingComparator(V4TestComparators.TRACKING) .as("Tracking should preserve LEAF_MANIFEST_WITH_DV metadata") @@ -1153,7 +1180,8 @@ public void rootMixedDataAndManifestFilesFirstRowIdAssignment() throws IOExcepti null, // not modified COMMIT_FIRST_ROW_ID, null, // no change bitmaps - null)); + null, + null)); // snapshot timestamp assertThat(files.get(1).tracking()) .usingComparator(V4TestComparators.TRACKING) .isEqualTo( @@ -1165,7 +1193,8 @@ public void rootMixedDataAndManifestFilesFirstRowIdAssignment() throws IOExcepti null, // not modified COMMIT_FIRST_ROW_ID + FILE_A.recordCount(), null, // no change bitmaps - null)); + null, + null)); // snapshot timestamp assertThat(files.get(2).tracking()) .usingComparator(V4TestComparators.TRACKING) .isEqualTo( @@ -1177,7 +1206,8 @@ public void rootMixedDataAndManifestFilesFirstRowIdAssignment() throws IOExcepti null, // not modified COMMIT_FIRST_ROW_ID + FILE_A.recordCount() + leafRowsAssigned, null, // no change bitmaps - null)); + null, + null)); // snapshot timestamp } } @@ -1215,7 +1245,8 @@ public void leafDataFileAppend() throws IOException { null, // not modified COMMIT_FIRST_ROW_ID, null, // no change bitmaps - null)); + null, + null)); // snapshot timestamp file = rawReadOne(in); assertThat(file) @@ -1233,7 +1264,8 @@ public void leafDataFileAppend() throws IOException { null, // not modified null, // inherit first row ID null, // no change bitmaps - null)); + null, + null)); // snapshot timestamp } @Test @@ -1270,7 +1302,8 @@ public void leafDataFileAppendAtDataSeq() throws IOException { null, // not modified COMMIT_FIRST_ROW_ID, null, // no change bitmaps - null)); + null, + null)); // snapshot timestamp file = rawReadOne(in); assertThat(file) @@ -1288,7 +1321,8 @@ public void leafDataFileAppendAtDataSeq() throws IOException { null, // not modified null, // inherit first row ID null, // no change bitmaps - null)); + null, + null)); // snapshot timestamp } @Test @@ -1325,7 +1359,8 @@ public void leafDataFileAppendWithoutSnapshotID() throws IOException { null, // added, not modified COMMIT_FIRST_ROW_ID, null, // no change bitmaps - null)); + null, + null)); // snapshot timestamp file = rawReadOne(in); assertThat(file) @@ -1343,7 +1378,8 @@ public void leafDataFileAppendWithoutSnapshotID() throws IOException { null, // not modified null, // inherit first row ID null, // no change bitmaps - null)); + null, + null)); // snapshot timestamp } @Test @@ -1380,7 +1416,8 @@ public void leafDataFileWithDVAppend() throws IOException { null, // added, not modified COMMIT_FIRST_ROW_ID, null, // no change bitmaps - null)); + null, + null)); // snapshot timestamp file = rawReadOne(in); assertThat(file) @@ -1398,7 +1435,8 @@ public void leafDataFileWithDVAppend() throws IOException { null, // added, not modified null, // inherit first row ID null, // no change bitmaps - null)); + null, + null)); // snapshot timestamp } @Test @@ -1436,7 +1474,8 @@ public void leafDataFileWithDVModify() throws IOException { COMMIT_SNAPSHOT_ID, originalTracking.firstRowId(), null, // no change bitmaps - null)); + null, + SNAPSHOT_TS)); } } @@ -1475,7 +1514,8 @@ public void leafDataFileExisting() throws IOException { originalTracking.modifiedSnapshotId(), originalTracking.firstRowId(), null, // no change bitmaps - null)); + null, + SNAPSHOT_TS)); } } @@ -1515,7 +1555,8 @@ public void leafDataFileDelete() throws IOException { originalTracking.modifiedSnapshotId(), originalTracking.firstRowId(), null, // no change bitmaps - null)); + null, + SNAPSHOT_TS)); } @Test @@ -1554,7 +1595,8 @@ public void leafDataFileReplace() throws IOException { originalTracking.modifiedSnapshotId(), originalTracking.firstRowId(), null, // no change bitmaps - null)); + null, + SNAPSHOT_TS)); } @Test @@ -1597,7 +1639,8 @@ public void leafMixedPartitionDataFiles() throws IOException { null, // not modified COMMIT_FIRST_ROW_ID, null, // no change bitmaps - null)); + null, + null)); // snapshot timestamp } @Test diff --git a/core/src/test/java/org/apache/iceberg/V4TestComparators.java b/core/src/test/java/org/apache/iceberg/V4TestComparators.java index d40a9bc46be5..f65c47234813 100644 --- a/core/src/test/java/org/apache/iceberg/V4TestComparators.java +++ b/core/src/test/java/org/apache/iceberg/V4TestComparators.java @@ -115,6 +115,7 @@ private static > Comparator natural() { .thenComparing(Tracking::dataSequenceNumber, natural()) .thenComparing(Tracking::fileSequenceNumber, natural()) .thenComparing(Tracking::modifiedSnapshotId, natural()) + .thenComparing(Tracking::snapshotTimestampMicros, natural()) .thenComparing(Tracking::firstRowId, natural()) .thenComparing(Tracking::deletedPositions, BYTES) .thenComparing(Tracking::replacedPositions, BYTES)); From ace86614c9039e634a2d5218df75389562dfc819 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Fri, 20 Nov 2026 20:49:22 -0800 Subject: [PATCH 2/3] API: rename ContentFile.snapshotTimestampMicros to lastUpdatedTimestampMicros The file-level API describes when the file was added or last changed in the table, so avoid referencing snapshots at this abstraction. Generated-by: Cursor Co-authored-by: Cursor --- api/src/main/java/org/apache/iceberg/ContentFile.java | 2 +- .../main/java/org/apache/iceberg/TrackedFileAdapters.java | 2 +- .../java/org/apache/iceberg/TestTrackedFileAdapters.java | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/ContentFile.java b/api/src/main/java/org/apache/iceberg/ContentFile.java index 708b2d375d38..da2483b3dc32 100644 --- a/api/src/main/java/org/apache/iceberg/ContentFile.java +++ b/api/src/main/java/org/apache/iceberg/ContentFile.java @@ -192,7 +192,7 @@ default Long fileSequenceNumber() { *

Returns null if the snapshot timestamp is unknown. This may happen for files in tables with * format version below 4. */ - default Long snapshotTimestampMicros() { + default Long lastUpdatedTimestampMicros() { return null; } diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java index 5e6acb5a8fa0..e065f012d7dc 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java @@ -149,7 +149,7 @@ public Long fileSequenceNumber() { } @Override - public Long snapshotTimestampMicros() { + public Long lastUpdatedTimestampMicros() { Tracking tracking = tracking(); return tracking != null ? tracking.snapshotTimestampMicros() : null; } diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java index 1794bba376aa..7d188884c79d 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java @@ -234,7 +234,7 @@ void dataFileAdapterDelegation() { assertThat(dataFile.sortOrderId()).isEqualTo(3); assertThat(dataFile.dataSequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER); assertThat(dataFile.fileSequenceNumber()).isEqualTo(FILE_SEQUENCE_NUMBER); - assertThat(dataFile.snapshotTimestampMicros()).isEqualTo(SNAPSHOT_TIMESTAMP_MICROS); + assertThat(dataFile.lastUpdatedTimestampMicros()).isEqualTo(SNAPSHOT_TIMESTAMP_MICROS); assertThat(dataFile.firstRowId()).isEqualTo(FIRST_ROW_ID); assertThat(dataFile.keyMetadata()).isEqualTo(ByteBuffer.wrap(new byte[] {1, 2, 3})); assertThat(dataFile.splitOffsets()).containsExactly(50L, 100L); @@ -347,7 +347,7 @@ void dvDeleteFileAdapterDelegation() { assertThat(dvFile.partition()).isSameAs(PARTITION); assertThat(dvFile.dataSequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER); assertThat(dvFile.fileSequenceNumber()).isEqualTo(FILE_SEQUENCE_NUMBER); - assertThat(dvFile.snapshotTimestampMicros()).isEqualTo(SNAPSHOT_TIMESTAMP_MICROS); + assertThat(dvFile.lastUpdatedTimestampMicros()).isEqualTo(SNAPSHOT_TIMESTAMP_MICROS); assertThat(dvFile.manifestLocation()).isEqualTo(MANIFEST_LOCATION); // fields that are null for DVs @@ -394,7 +394,8 @@ private static void assertSameDataFile(DataFile actual, DataFile expected) { assertThat(actual.sortOrderId()).isEqualTo(expected.sortOrderId()); assertThat(actual.dataSequenceNumber()).isEqualTo(expected.dataSequenceNumber()); assertThat(actual.fileSequenceNumber()).isEqualTo(expected.fileSequenceNumber()); - assertThat(actual.snapshotTimestampMicros()).isEqualTo(expected.snapshotTimestampMicros()); + assertThat(actual.lastUpdatedTimestampMicros()) + .isEqualTo(expected.lastUpdatedTimestampMicros()); assertThat(actual.firstRowId()).isEqualTo(expected.firstRowId()); assertThat(actual.pos()).isEqualTo(expected.pos()); assertThat(actual.manifestLocation()).isEqualTo(expected.manifestLocation()); From f7db95639073540949a9358c8eb92cb4f6fa3b3f Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Fri, 20 Nov 2026 20:55:12 -0800 Subject: [PATCH 3/3] Core: pass snapshot timestamp to TrackingStruct in readManifestLocations test Generated-by: Cursor Co-authored-by: Cursor --- .../org/apache/iceberg/TestV4ManifestReader.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index 3dab3010921b..0258b265a3c5 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -2066,29 +2066,31 @@ public void resolutionResolvesOnlyRelativePaths(FileFormat format) throws IOExce public void readManifestLocations(FileFormat format) throws IOException { TrackedFile dataFile = dataFile( - new TrackingStruct(EntryStatus.ADDED, SNAPSHOT_ID, null, null, null, null, null, null), + new TrackingStruct( + EntryStatus.ADDED, SNAPSHOT_ID, null, null, null, null, null, null, null), "s3://bucket/table/data-file.parquet"); TrackedFile deletedDataManifest = V4TestHelpers.manifestRefWithTracking( new TrackingStruct( - EntryStatus.DELETED, SNAPSHOT_ID, null, null, null, null, null, null), + EntryStatus.DELETED, SNAPSHOT_ID, null, null, null, null, null, null, null), FileContent.DATA_MANIFEST, "s3://bucket/table/deleted-data-manifest.parquet"); TrackedFile dataManifest = V4TestHelpers.manifestRefWithTracking( - new TrackingStruct(EntryStatus.ADDED, SNAPSHOT_ID, null, null, null, null, null, null), + new TrackingStruct( + EntryStatus.ADDED, SNAPSHOT_ID, null, null, null, null, null, null, null), FileContent.DATA_MANIFEST, "s3://bucket/table/data-manifest.parquet"); TrackedFile deletedDeleteManifest = V4TestHelpers.manifestRefWithTracking( new TrackingStruct( - EntryStatus.DELETED, SNAPSHOT_ID, null, null, null, null, null, null), + EntryStatus.DELETED, SNAPSHOT_ID, null, null, null, null, null, null, null), FileContent.DELETE_MANIFEST, "s3://bucket/table/deleted-delete-manifest.parquet"); TrackedFile deleteManifest = V4TestHelpers.manifestRefWithTracking( new TrackingStruct( - EntryStatus.EXISTING, SNAPSHOT_ID, null, null, null, null, null, null), + EntryStatus.EXISTING, SNAPSHOT_ID, null, null, null, null, null, null, null), FileContent.DELETE_MANIFEST, "s3://bucket/table/delete-manifest.parquet");