Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions api/src/main/java/org/apache/iceberg/ContentFile.java
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,19 @@ default Long fileSequenceNumber() {
return null;
}

/**
* Returns the microsecond timestamp of the snapshot that added or changed this file.
*
* <p>The value is the snapshot's {@code timestamp-ms} converted to microseconds and is assigned
* at commit.
*
* <p>Returns null if the snapshot timestamp is unknown. This may happen for files in tables with
* format version below 4.
*/
default Long lastUpdatedTimestampMicros() {
return null;
}

/**
* Returns the starting row ID to assign to new rows in the data file (with _row_id set to null).
*/
Expand Down
5 changes: 5 additions & 0 deletions api/src/main/java/org/apache/iceberg/ManifestFile.java
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,11 @@ default Long firstRowId() {
return null;
}

/** Returns the microsecond timestamp of the snapshot that added the manifest file. */
default Long snapshotTimestampMicros() {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should this one just be a long? For a manifest file wouldn't we always know this (regardless of inheritance)?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For v4 tables (green or upgraded), their manifest files (committed before or after the upgrade) would always have it via inheritance.

for v1-v3 tables, their manifest files won't have this. Are you thinking about wiring in the snapshot timestamp inheritance for v1-v3 tables too? and don't expose the metadata column for v1-v3 tables?

@amogh-jahagirdar amogh-jahagirdar Nov 18, 2026 •

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah yeah sorry I was thinking from a pure v4 mindset, for the older v1-v3 manifests you're right they won't have this and we can't derive those because the snapshots for when those were written may have been expired. We could fill in a sentinel value (unix epoch) but that seems worse and error prone to interpretation when later materialized (compared to just leveraging the snapshot post v4 upgrade). So I think I'd leave it as is then

return null;
}

/** Returns the manifest deletion vector, or null if absent. */
default ManifestBitmap manifestDeletionVector() {
return null;
Expand Down
12 changes: 10 additions & 2 deletions core/src/main/java/org/apache/iceberg/MetadataColumns.java
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,12 @@ private MetadataColumns() {}
"_last_updated_sequence_number",
Types.LongType.get(),
"Sequence number when the row was last updated");
public static final NestedField LAST_UPDATED_TIMESTAMP =
NestedField.optional(
Integer.MAX_VALUE - 109,
"_last_updated_timestamp",
Types.TimestampType.withZone(),
"Snapshot timestamp when the row was last updated");

private static final Map<String, NestedField> META_COLUMNS =
ImmutableMap.of(
Expand All @@ -114,7 +120,8 @@ private MetadataColumns() {}
IS_DELETED.name(), IS_DELETED,
SPEC_ID.name(), SPEC_ID,
ROW_ID.name(), ROW_ID,
LAST_UPDATED_SEQUENCE_NUMBER.name(), LAST_UPDATED_SEQUENCE_NUMBER);
LAST_UPDATED_SEQUENCE_NUMBER.name(), LAST_UPDATED_SEQUENCE_NUMBER,
LAST_UPDATED_TIMESTAMP.name(), LAST_UPDATED_TIMESTAMP);

private static final Set<Integer> META_IDS =
ImmutableSet.of(
Expand All @@ -124,7 +131,8 @@ private MetadataColumns() {}
SPEC_ID.fieldId(),
PARTITION_COLUMN_ID,
ROW_ID.fieldId(),
LAST_UPDATED_SEQUENCE_NUMBER.fieldId());
LAST_UPDATED_SEQUENCE_NUMBER.fieldId(),
LAST_UPDATED_TIMESTAMP.fieldId());

public static Set<Integer> metadataFieldIds() {
return META_IDS;
Expand Down
19 changes: 18 additions & 1 deletion core/src/main/java/org/apache/iceberg/RootManifestFile.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.relocated.com.google.common.base.MoreObjects;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
Expand All @@ -42,6 +43,7 @@ static RootManifestFile from(Snapshot snapshot, String tableLocation, FileIO io)
io.newInputFile(location).getLength(),
snapshot.snapshotId(),
snapshot.sequenceNumber(),
TimeUnit.MILLISECONDS.toMicros(snapshot.timestampMillis()),
snapshot.firstRowId(),
/* keyMetadata= */ null);
}
Expand All @@ -50,6 +52,7 @@ static RootManifestFile from(Snapshot snapshot, String tableLocation, FileIO io)
private final long length;
private final long snapshotId;
private final long sequenceNumber;
private final long snapshotTimestampMicros;
private final Long firstRowId;
private final byte[] keyMetadata;

Expand All @@ -58,13 +61,15 @@ static RootManifestFile from(Snapshot snapshot, String tableLocation, FileIO io)
long length,
long snapshotId,
long sequenceNumber,
long snapshotTimestampMicros,
Long firstRowId,
ByteBuffer keyMetadata) {
Preconditions.checkArgument(location != null, "Invalid location: null");
this.location = location;
this.length = length;
this.snapshotId = snapshotId;
this.sequenceNumber = sequenceNumber;
this.snapshotTimestampMicros = snapshotTimestampMicros;
this.firstRowId = firstRowId;
this.keyMetadata = ByteBuffers.toByteArray(keyMetadata);
}
Expand Down Expand Up @@ -144,6 +149,11 @@ public ByteBuffer keyMetadata() {
return keyMetadata == null ? null : ByteBuffer.wrap(keyMetadata);
}

@Override
public Long snapshotTimestampMicros() {
return snapshotTimestampMicros;
}

@Override
public Long firstRowId() {
return firstRowId;
Expand All @@ -161,7 +171,13 @@ public ManifestFile copy() {
? null
: ByteBuffer.wrap(Arrays.copyOf(keyMetadata, keyMetadata.length));
return new RootManifestFile(
location, length, snapshotId, sequenceNumber, firstRowId, keyMetadataCopy);
location,
length,
snapshotId,
sequenceNumber,
snapshotTimestampMicros,
firstRowId,
keyMetadataCopy);
}

@Override
Expand All @@ -171,6 +187,7 @@ public String toString() {
.add("length", length)
.add("snapshot_id", snapshotId)
.add("sequence_number", sequenceNumber)
.add("snapshot_timestamp_micros", snapshotTimestampMicros)
.add("first_row_id", firstRowId)
.add("key_metadata", keyMetadata == null ? "null" : "(redacted)")
.toString();
Expand Down
6 changes: 5 additions & 1 deletion core/src/main/java/org/apache/iceberg/StatsUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ private StatsUtil() {}

private static final int LAST_UPDATED_SEQ_NUM_BASE_ID = 9_000;
private static final int ROW_ID_BASE_ID = 9_200;
private static final int LAST_UPDATED_TIMESTAMP_BASE_ID = 9_400;

// Offsets used for individual stats columns
static final int LOWER_BOUND_OFFSET = 1;
Expand Down Expand Up @@ -83,7 +84,8 @@ private StatsUtil() {}
private static final Map<Integer, Integer> METADATA_ID_TO_BASE_ID =
ImmutableMap.of(
MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId(), LAST_UPDATED_SEQ_NUM_BASE_ID,
MetadataColumns.ROW_ID.fieldId(), ROW_ID_BASE_ID);
MetadataColumns.ROW_ID.fieldId(), ROW_ID_BASE_ID,
MetadataColumns.LAST_UPDATED_TIMESTAMP.fieldId(), LAST_UPDATED_TIMESTAMP_BASE_ID);

/**
* Return the base ID of the stats struct for the given field ID, or -1 if the ID is out of range.
Expand Down Expand Up @@ -116,6 +118,8 @@ static int toFieldId(int statId) {
return MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId();
} else if (inBaseIdRange(ROW_ID_BASE_ID, statId)) {
return MetadataColumns.ROW_ID.fieldId();
} else if (inBaseIdRange(LAST_UPDATED_TIMESTAMP_BASE_ID, statId)) {
return MetadataColumns.LAST_UPDATED_TIMESTAMP.fieldId();
} else {
throw new UnsupportedOperationException("Unsupported metadata stats field ID: " + statId);
}
Expand Down
21 changes: 21 additions & 0 deletions core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,12 @@ public Long fileSequenceNumber() {
Tracking tracking = tracking();
return tracking != null ? tracking.fileSequenceNumber() : null;
}

@Override
public Long lastUpdatedTimestampMicros() {
Tracking tracking = tracking();
return tracking != null ? tracking.snapshotTimestampMicros() : null;
}
}

/**
Expand Down Expand Up @@ -533,6 +539,11 @@ public Long firstRowId() {
return file.tracking() != null ? file.tracking().firstRowId() : null;
}

@Override
public Long snapshotTimestampMicros() {
return file.tracking().snapshotTimestampMicros();
}

@Override
public ManifestBitmap manifestDeletionVector() {
return file.manifestInfo().manifestDeletionVector();
Expand Down Expand Up @@ -957,6 +968,11 @@ public ByteBuffer replacedPositions() {
return null;
}

@Override
public Long snapshotTimestampMicros() {
return null;
}

@Override
public String manifestLocation() {
return entry.file().manifestLocation();
Expand Down Expand Up @@ -1023,6 +1039,11 @@ public ByteBuffer replacedPositions() {
return null;
}

@Override
public Long snapshotTimestampMicros() {
return manifest.snapshotTimestampMicros();
}

@Override
public String manifestLocation() {
return null;
Expand Down
15 changes: 14 additions & 1 deletion core/src/main/java/org/apache/iceberg/Tracking.java
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,12 @@ interface Tracking {
"replaced_positions",
Types.BinaryType.get(),
"Bitmap of positions replaced in this snapshot");
Types.NestedField SNAPSHOT_TIMESTAMP =
Types.NestedField.optional(
8,
"snapshot_timestamp",
Types.TimestampType.withZone(),

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

withZone?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, see the spec discussion here: #18526 (comment)

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

k 🤷

"Timestamp in microseconds of the snapshot that added or last modified the file's data");

static Types.StructType schema() {
return Types.StructType.of(
Expand All @@ -75,7 +81,8 @@ static Types.StructType schema() {
MODIFIED_SNAPSHOT_ID,
FIRST_ROW_ID,
DELETED_POSITIONS,
REPLACED_POSITIONS);
REPLACED_POSITIONS,
SNAPSHOT_TIMESTAMP);
}

/** Returns the status of the entry. */
Expand Down Expand Up @@ -107,6 +114,12 @@ default boolean isLive() {
/** Returns the bitmap of positions replaced in this snapshot. */
ByteBuffer replacedPositions();

/**
* Returns the timestamp in microseconds of the snapshot that added or last modified the file's
* data.
*/
Long snapshotTimestampMicros();

/** Returns the manifest location this entry was read from, or null. */
String manifestLocation();

Expand Down
11 changes: 9 additions & 2 deletions core/src/main/java/org/apache/iceberg/TrackingBuilder.java
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ static Tracking replaced(Tracking source, long newSnapshotId) {
private Long modifiedSnapshotId;
private byte[] deletedPositions;
private byte[] replacedPositions;
private Long snapshotTimestampMicros;

private TrackingBuilder(Long newSnapshotId) {
this.status = EntryStatus.ADDED;
Expand All @@ -97,6 +98,7 @@ private TrackingBuilder(Long newSnapshotId) {
this.modifiedSnapshotId = null;
this.deletedPositions = null;
this.replacedPositions = null;
this.snapshotTimestampMicros = null;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When a file is added with an older sequence number (appendAtSeq, used by compaction), it keeps the old sequence number, but its timestamp is null here, so it gets the new commit's time. The two values then disagree: the rows look old by sequence number but new by timestamp. Should these files keep an older timestamp too? #18526 requires null for all ADDED entries, so this would need a spec change as well.

@stevenzwu stevenzwu Nov 21, 2026 •

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Rewrites must copy non-null _last_updated_sequence_number and _last_updated_timestamp forward in data files, so compacted rows don't inherit either value, and ADDED entries can stay null as #18526 specifies.

}

private TrackingBuilder(long newSnapshotId, long dataSequenceNumber) {
Expand All @@ -116,6 +118,7 @@ private TrackingBuilder(Tracking source, long newSnapshotId) {
this.modifiedSnapshotId = source.modifiedSnapshotId();
this.deletedPositions = null;
this.replacedPositions = null;
this.snapshotTimestampMicros = source.snapshotTimestampMicros();
}

/** Indicates that the DV has been updated for the new Tracking. */
Expand Down Expand Up @@ -145,6 +148,8 @@ TrackingBuilder columnFilesUpdated() {
// dataSequenceNumber as writers are required to rewrite v2 equality and position deletes to DVs
// when applying column update.
this.dataSequenceNumber = null;
// Reset to null to inherit from the new snapshot timestamp because the file's data changed.
this.snapshotTimestampMicros = null;
return this;
}

Expand Down Expand Up @@ -177,7 +182,8 @@ Tracking build() {
modifiedSnapshotId,
firstRowId,
deletedPositions,
replacedPositions);
replacedPositions,
snapshotTimestampMicros);
}

private static Tracking terminal(EntryStatus to, Tracking source, long newSnapshotId) {
Expand All @@ -191,7 +197,8 @@ private static Tracking terminal(EntryStatus to, Tracking source, long newSnapsh
source.modifiedSnapshotId(),
source.firstRowId(),
null,
null);
null,
source.snapshotTimestampMicros());
}

private static void validateSource(Tracking source) {
Expand Down
22 changes: 19 additions & 3 deletions core/src/main/java/org/apache/iceberg/TrackingStruct.java
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ class TrackingStruct extends SupportsIndexProjection implements Tracking, Serial
Tracking.FIRST_ROW_ID,
Tracking.DELETED_POSITIONS,
Tracking.REPLACED_POSITIONS,
Tracking.SNAPSHOT_TIMESTAMP,
MetadataColumns.FILE_PATH,
MetadataColumns.ROW_POSITION);

Expand All @@ -62,6 +63,7 @@ class TrackingStruct extends SupportsIndexProjection implements Tracking, Serial
private Long firstRowId = null;
private byte[] deletedPositions = null;
private byte[] replacedPositions = null;
private Long snapshotTimestampMicros = null;

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note that inherit(long, long) below doesn't fill snapshotTimestampMicros yet. That's intentional: this PR is limited to the schema, interface, and struct changes.

Read inheritance comes in the next stacked PR. There, inherit takes the manifest's snapshot timestamp as a third argument and fills it under the same condition as the data sequence number, and the root manifest writer persists the commit timestamp for added and modified entries.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The spec in #18526 says that when writing any manifest, a new (ADDED) entry "must have a null snapshot_timestamp". It doesn't make an exception for the root manifest. But your plan above says the root manifest writer will write the commit time for new entries. For sequence numbers, the spec only asks for null in leaf manifests, so the root writer can fill them in. Should the timestamp rule allow the same, or should the root writer leave it null?


// set by manifest readers, not written to manifests
private String manifestLocation = null;
Expand Down Expand Up @@ -92,6 +94,7 @@ private TrackingStruct(TrackingStruct toCopy) {
toCopy.replacedPositions != null
? Arrays.copyOf(toCopy.replacedPositions, toCopy.replacedPositions.length)
: null;
this.snapshotTimestampMicros = toCopy.snapshotTimestampMicros;
this.manifestLocation = toCopy.manifestLocation;
this.manifestPos = toCopy.manifestPos;
}
Expand All @@ -104,7 +107,8 @@ private TrackingStruct(TrackingStruct toCopy) {
Long modifiedSnapshotId,
Long firstRowId,
byte[] deletedPositions,
byte[] replacedPositions) {
byte[] replacedPositions,
Long snapshotTimestampMicros) {
super(BASE_TYPE.fields().size());
this.status = status;
this.snapshotId = snapshotId;
Expand All @@ -114,6 +118,7 @@ private TrackingStruct(TrackingStruct toCopy) {
this.firstRowId = firstRowId;
this.deletedPositions = deletedPositions;
this.replacedPositions = replacedPositions;
this.snapshotTimestampMicros = snapshotTimestampMicros;
}

/**
Expand Down Expand Up @@ -220,6 +225,11 @@ public ByteBuffer replacedPositions() {
return replacedPositions != null ? ByteBuffer.wrap(replacedPositions) : null;
}

@Override
public Long snapshotTimestampMicros() {
return snapshotTimestampMicros;
}

@Override
public String manifestLocation() {
return manifestLocation;
Expand Down Expand Up @@ -259,8 +269,10 @@ private Object getByPos(int pos) {
case 7:
return replacedPositions();
case 8:
return manifestLocation;
return snapshotTimestampMicros;
case 9:
return manifestLocation;
case 10:
return manifestPos;
default:
throw new UnsupportedOperationException("Unknown field ordinal: " + pos);
Expand Down Expand Up @@ -295,9 +307,12 @@ protected <T> void internalSet(int pos, T value) {
this.replacedPositions = ByteBuffers.toByteArray((ByteBuffer) value);
break;
case 8:
this.manifestLocation = (String) value;
this.snapshotTimestampMicros = (Long) value;
break;
case 9:
this.manifestLocation = (String) value;
break;
case 10:
this.manifestPos = (long) value;
break;
default:
Expand All @@ -316,6 +331,7 @@ public String toString() {
.add("first_row_id", firstRowId)
.add("deleted_positions", deletedPositions == null ? "null" : "(binary)")
.add("replaced_positions", replacedPositions == null ? "null" : "(binary)")
.add("snapshot_timestamp_micros", snapshotTimestampMicros)
.toString();
}
}
Loading
Loading