Repository navigation
Core, API: add snapshot timestamp tracking and _last_updated_timestamp metadata column #19019
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
e550caf
ace8661
f7db956
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -65,6 +65,12 @@ interface Tracking { | |
| "replaced_positions", | ||
| Types.BinaryType.get(), | ||
| "Bitmap of positions replaced in this snapshot"); | ||
| Types.NestedField SNAPSHOT_TIMESTAMP = | ||
| Types.NestedField.optional( | ||
| 8, | ||
| "snapshot_timestamp", | ||
| Types.TimestampType.withZone(), | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. withZone?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yes, see the spec discussion here: #18526 (comment)
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. k 🤷 |
||
| "Timestamp in microseconds of the snapshot that added or last modified the file's data"); | ||
|
|
||
| static Types.StructType schema() { | ||
| return Types.StructType.of( | ||
|
|
@@ -75,7 +81,8 @@ static Types.StructType schema() { | |
| MODIFIED_SNAPSHOT_ID, | ||
| FIRST_ROW_ID, | ||
| DELETED_POSITIONS, | ||
| REPLACED_POSITIONS); | ||
| REPLACED_POSITIONS, | ||
| SNAPSHOT_TIMESTAMP); | ||
| } | ||
|
|
||
| /** Returns the status of the entry. */ | ||
|
|
@@ -107,6 +114,12 @@ default boolean isLive() { | |
| /** Returns the bitmap of positions replaced in this snapshot. */ | ||
| ByteBuffer replacedPositions(); | ||
|
|
||
| /** | ||
| * Returns the timestamp in microseconds of the snapshot that added or last modified the file's | ||
| * data. | ||
| */ | ||
| Long snapshotTimestampMicros(); | ||
|
|
||
| /** Returns the manifest location this entry was read from, or null. */ | ||
| String manifestLocation(); | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -86,6 +86,7 @@ static Tracking replaced(Tracking source, long newSnapshotId) { | |
| private Long modifiedSnapshotId; | ||
| private byte[] deletedPositions; | ||
| private byte[] replacedPositions; | ||
| private Long snapshotTimestampMicros; | ||
|
|
||
| private TrackingBuilder(Long newSnapshotId) { | ||
| this.status = EntryStatus.ADDED; | ||
|
|
@@ -97,6 +98,7 @@ private TrackingBuilder(Long newSnapshotId) { | |
| this.modifiedSnapshotId = null; | ||
| this.deletedPositions = null; | ||
| this.replacedPositions = null; | ||
| this.snapshotTimestampMicros = null; | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. When a file is added with an older sequence number (
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Rewrites must copy non-null |
||
| } | ||
|
|
||
| private TrackingBuilder(long newSnapshotId, long dataSequenceNumber) { | ||
|
|
@@ -116,6 +118,7 @@ private TrackingBuilder(Tracking source, long newSnapshotId) { | |
| this.modifiedSnapshotId = source.modifiedSnapshotId(); | ||
| this.deletedPositions = null; | ||
| this.replacedPositions = null; | ||
| this.snapshotTimestampMicros = source.snapshotTimestampMicros(); | ||
| } | ||
|
|
||
| /** Indicates that the DV has been updated for the new Tracking. */ | ||
|
|
@@ -145,6 +148,8 @@ TrackingBuilder columnFilesUpdated() { | |
| // dataSequenceNumber as writers are required to rewrite v2 equality and position deletes to DVs | ||
| // when applying column update. | ||
| this.dataSequenceNumber = null; | ||
| // Reset to null to inherit from the new snapshot timestamp because the file's data changed. | ||
| this.snapshotTimestampMicros = null; | ||
| return this; | ||
| } | ||
|
|
||
|
|
@@ -177,7 +182,8 @@ Tracking build() { | |
| modifiedSnapshotId, | ||
| firstRowId, | ||
| deletedPositions, | ||
| replacedPositions); | ||
| replacedPositions, | ||
| snapshotTimestampMicros); | ||
| } | ||
|
|
||
| private static Tracking terminal(EntryStatus to, Tracking source, long newSnapshotId) { | ||
|
|
@@ -191,7 +197,8 @@ private static Tracking terminal(EntryStatus to, Tracking source, long newSnapsh | |
| source.modifiedSnapshotId(), | ||
| source.firstRowId(), | ||
| null, | ||
| null); | ||
| null, | ||
| source.snapshotTimestampMicros()); | ||
| } | ||
|
|
||
| private static void validateSource(Tracking source) { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -39,6 +39,7 @@ class TrackingStruct extends SupportsIndexProjection implements Tracking, Serial | |
| Tracking.FIRST_ROW_ID, | ||
| Tracking.DELETED_POSITIONS, | ||
| Tracking.REPLACED_POSITIONS, | ||
| Tracking.SNAPSHOT_TIMESTAMP, | ||
| MetadataColumns.FILE_PATH, | ||
| MetadataColumns.ROW_POSITION); | ||
|
|
||
|
|
@@ -62,6 +63,7 @@ class TrackingStruct extends SupportsIndexProjection implements Tracking, Serial | |
| private Long firstRowId = null; | ||
| private byte[] deletedPositions = null; | ||
| private byte[] replacedPositions = null; | ||
| private Long snapshotTimestampMicros = null; | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Note that Read inheritance comes in the next stacked PR. There,
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The spec in #18526 says that when writing any manifest, a new (ADDED) entry "must have a null snapshot_timestamp". It doesn't make an exception for the root manifest. But your plan above says the root manifest writer will write the commit time for new entries. For sequence numbers, the spec only asks for null in leaf manifests, so the root writer can fill them in. Should the timestamp rule allow the same, or should the root writer leave it null? |
||
|
|
||
| // set by manifest readers, not written to manifests | ||
| private String manifestLocation = null; | ||
|
|
@@ -92,6 +94,7 @@ private TrackingStruct(TrackingStruct toCopy) { | |
| toCopy.replacedPositions != null | ||
| ? Arrays.copyOf(toCopy.replacedPositions, toCopy.replacedPositions.length) | ||
| : null; | ||
| this.snapshotTimestampMicros = toCopy.snapshotTimestampMicros; | ||
| this.manifestLocation = toCopy.manifestLocation; | ||
| this.manifestPos = toCopy.manifestPos; | ||
| } | ||
|
|
@@ -104,7 +107,8 @@ private TrackingStruct(TrackingStruct toCopy) { | |
| Long modifiedSnapshotId, | ||
| Long firstRowId, | ||
| byte[] deletedPositions, | ||
| byte[] replacedPositions) { | ||
| byte[] replacedPositions, | ||
| Long snapshotTimestampMicros) { | ||
| super(BASE_TYPE.fields().size()); | ||
| this.status = status; | ||
| this.snapshotId = snapshotId; | ||
|
|
@@ -114,6 +118,7 @@ private TrackingStruct(TrackingStruct toCopy) { | |
| this.firstRowId = firstRowId; | ||
| this.deletedPositions = deletedPositions; | ||
| this.replacedPositions = replacedPositions; | ||
| this.snapshotTimestampMicros = snapshotTimestampMicros; | ||
| } | ||
|
|
||
| /** | ||
|
|
@@ -220,6 +225,11 @@ public ByteBuffer replacedPositions() { | |
| return replacedPositions != null ? ByteBuffer.wrap(replacedPositions) : null; | ||
| } | ||
|
|
||
| @Override | ||
| public Long snapshotTimestampMicros() { | ||
| return snapshotTimestampMicros; | ||
| } | ||
|
|
||
| @Override | ||
| public String manifestLocation() { | ||
| return manifestLocation; | ||
|
|
@@ -259,8 +269,10 @@ private Object getByPos(int pos) { | |
| case 7: | ||
| return replacedPositions(); | ||
| case 8: | ||
| return manifestLocation; | ||
| return snapshotTimestampMicros; | ||
| case 9: | ||
| return manifestLocation; | ||
| case 10: | ||
| return manifestPos; | ||
| default: | ||
| throw new UnsupportedOperationException("Unknown field ordinal: " + pos); | ||
|
|
@@ -295,9 +307,12 @@ protected <T> void internalSet(int pos, T value) { | |
| this.replacedPositions = ByteBuffers.toByteArray((ByteBuffer) value); | ||
| break; | ||
| case 8: | ||
| this.manifestLocation = (String) value; | ||
| this.snapshotTimestampMicros = (Long) value; | ||
| break; | ||
| case 9: | ||
| this.manifestLocation = (String) value; | ||
| break; | ||
| case 10: | ||
| this.manifestPos = (long) value; | ||
| break; | ||
| default: | ||
|
|
@@ -316,6 +331,7 @@ public String toString() { | |
| .add("first_row_id", firstRowId) | ||
| .add("deleted_positions", deletedPositions == null ? "null" : "(binary)") | ||
| .add("replaced_positions", replacedPositions == null ? "null" : "(binary)") | ||
| .add("snapshot_timestamp_micros", snapshotTimestampMicros) | ||
| .toString(); | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
should this one just be a
long? For a manifest file wouldn't we always know this (regardless of inheritance)?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
For v4 tables (green or upgraded), their manifest files (committed before or after the upgrade) would always have it via inheritance.
for v1-v3 tables, their manifest files won't have this. Are you thinking about wiring in the snapshot timestamp inheritance for v1-v3 tables too? and don't expose the metadata column for v1-v3 tables?
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Ah yeah sorry I was thinking from a pure v4 mindset, for the older v1-v3 manifests you're right they won't have this and we can't derive those because the snapshots for when those were written may have been expired. We could fill in a sentinel value (unix epoch) but that seems worse and error prone to interpretation when later materialized (compared to just leveraging the snapshot post v4 upgrade). So I think I'd leave it as is then