From 29970b9c12f57efe031d5015a16d2860a399e05f Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Tue, 10 Mar 2026 13:32:54 -0700 Subject: [PATCH 1/5] Core: make sure snapshot timestamp is monotonically increasing for V4 tables. Made-with: Cursor Model: claude-4.6-opus-high-thinking Co-authored-by: Cursor --- .../org/apache/iceberg/SnapshotProducer.java | 29 +++++++++++- .../org/apache/iceberg/TableMetadata.java | 1 + .../apache/iceberg/TestSnapshotProducer.java | 46 +++++++++++++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/iceberg/SnapshotProducer.java b/core/src/main/java/org/apache/iceberg/SnapshotProducer.java index d97c63b61608..63d97f3b15ec 100644 --- a/core/src/main/java/org/apache/iceberg/SnapshotProducer.java +++ b/core/src/main/java/org/apache/iceberg/SnapshotProducer.java @@ -35,6 +35,7 @@ import com.github.benmanes.caffeine.cache.LoadingCache; import java.io.IOException; import java.math.RoundingMode; +import java.time.Clock; import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -119,6 +120,7 @@ public void accept(String file) { private SnapshotAncestryValidator snapshotAncestryValidator = SnapshotAncestryValidator.NON_VALIDATING; + private Clock clock = Clock.systemUTC(); private ExecutorService workerPool; private ExecutorService writePool; private int writePoolParallelism = ThreadPools.WORKER_THREAD_POOL_SIZE; @@ -358,7 +360,7 @@ public Snapshot apply() { sequenceNumber, snapshotId(), parentSnapshotId, - System.currentTimeMillis(), + snapshotTimestampMillis(parentSnapshot), operation(), summary(base), base.currentSchemaId(), @@ -683,6 +685,31 @@ protected ManifestReader newDeleteManifestReader(ManifestFile manife return ManifestFiles.readDeleteManifest(manifest, ops.io(), ops.current().specsById()); } + @VisibleForTesting + void setClock(Clock newClock) { + this.clock = newClock; + } + + /** + * Generates the snapshot timestamp in milliseconds. + * + *

For format version 4 and above, this implements the Lamport clock algorithm to guarantee + * monotonically increasing snapshot timestamps. For older format versions, this returns the + * current wall clock time. + * + * @param parentSnapshot the parent snapshot on the target branch, or null if there is no parent + * @return the snapshot timestamp in milliseconds + */ + private long snapshotTimestampMillis(Snapshot parentSnapshot) { + long now = clock.millis(); + if (base.formatVersion() >= TableMetadata.MIN_FORMAT_VERSION_MONOTONIC_TIMESTAMPS + && parentSnapshot != null) { + return Math.max(now, parentSnapshot.timestampMillis() + 1); + } + + return now; + } + protected long snapshotId() { if (snapshotId == null) { synchronized (this) { diff --git a/core/src/main/java/org/apache/iceberg/TableMetadata.java b/core/src/main/java/org/apache/iceberg/TableMetadata.java index 07f6d3342727..1fdc2ce70cf1 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadata.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadata.java @@ -59,6 +59,7 @@ public class TableMetadata implements Serializable { static final int MIN_FORMAT_VERSION_ROW_LINEAGE = 3; static final int MIN_FORMAT_VERSION_PARQUET_MANIFESTS = 4; static final int MIN_FORMAT_VERSION_OPTIONAL_LOCATION = 4; + static final int MIN_FORMAT_VERSION_MONOTONIC_TIMESTAMPS = 4; static final int INITIAL_SPEC_ID = 0; static final int INITIAL_SORT_ORDER_ID = 1; static final int INITIAL_SCHEMA_ID = 0; diff --git a/core/src/test/java/org/apache/iceberg/TestSnapshotProducer.java b/core/src/test/java/org/apache/iceberg/TestSnapshotProducer.java index 1e6021a5c9fc..f2019dbcdd59 100644 --- a/core/src/test/java/org/apache/iceberg/TestSnapshotProducer.java +++ b/core/src/test/java/org/apache/iceberg/TestSnapshotProducer.java @@ -27,6 +27,9 @@ import java.io.File; import java.io.IOException; import java.nio.file.Paths; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -276,4 +279,47 @@ public void testWriteManifestsWithInvalidParallelismThrows() { executor.shutdownNow(); } } + + @TestTemplate + public void testSnapshotTimestampsAreMonotonicallyIncreasing() { + assumeThat(formatVersion) + .isGreaterThanOrEqualTo(TableMetadata.MIN_FORMAT_VERSION_MONOTONIC_TIMESTAMPS); + + table.newFastAppend().appendFile(FILE_A).commit(); + Snapshot first = table.currentSnapshot(); + + table.newFastAppend().appendFile(FILE_B).commit(); + Snapshot second = table.currentSnapshot(); + assertThat(second.timestampMillis()) + .as("V4 snapshot timestamps must be strictly increasing") + .isGreaterThan(first.timestampMillis()); + + table.newFastAppend().appendFile(FILE_C).commit(); + Snapshot third = table.currentSnapshot(); + assertThat(third.timestampMillis()) + .as("V4 snapshot timestamps must be strictly increasing") + .isGreaterThan(second.timestampMillis()); + } + + @TestTemplate + public void testV4LamportClockFastForwardsDriftedClock() { + assumeThat(formatVersion) + .isGreaterThanOrEqualTo(TableMetadata.MIN_FORMAT_VERSION_MONOTONIC_TIMESTAMPS); + + table.newFastAppend().appendFile(FILE_A).commit(); + long firstTs = table.currentSnapshot().timestampMillis(); + + // simulate clock drift: wall clock reports a time far in the past + long driftedTime = firstTs - 10_000; + Clock driftedClock = Clock.fixed(Instant.ofEpochMilli(driftedTime), ZoneOffset.UTC); + AppendFiles append = table.newFastAppend().appendFile(FILE_B); + ((SnapshotProducer) append).setClock(driftedClock); + append.commit(); + + long secondTs = table.currentSnapshot().timestampMillis(); + assertThat(secondTs) + .as( + "Lamport clock should fast-forward past the drifted wall clock to the last snapshot timestamp + 1 ms") + .isEqualTo(firstTs + 1); + } } From 9d3744e558f1fb5086f16c29685df18bb84e6232 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 11 May 2026 22:39:07 -0700 Subject: [PATCH 2/5] Core: test that V4 snapshot timestamp monotonicity is scoped to the target branch Pins that SnapshotProducer.snapshotTimestampMillis uses the parent on the target branch (via SnapshotUtil.latestSnapshot(base, targetBranch)) rather than the table's globally newest snapshot. The test commits to a separate branch with a clock pinned far in the future, then commits to main with a clock pinned just slightly past main's head. Main's new timestamp must equal the pinned wall-clock value (no fast-forward) and remain well below the sibling branch's head timestamp. Without this test, a regression that read base.currentSnapshot() (or any max-over-branches) for the parent would silently pass the existing single-branch monotonicity tests. Made-with: Cursor Model: claude-4.7-opus --- .../apache/iceberg/TestSnapshotProducer.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/core/src/test/java/org/apache/iceberg/TestSnapshotProducer.java b/core/src/test/java/org/apache/iceberg/TestSnapshotProducer.java index f2019dbcdd59..802d3ed6c386 100644 --- a/core/src/test/java/org/apache/iceberg/TestSnapshotProducer.java +++ b/core/src/test/java/org/apache/iceberg/TestSnapshotProducer.java @@ -322,4 +322,46 @@ public void testV4LamportClockFastForwardsDriftedClock() { "Lamport clock should fast-forward past the drifted wall clock to the last snapshot timestamp + 1 ms") .isEqualTo(firstTs + 1); } + + @TestTemplate + public void testV4MonotonicityIsScopedToTargetBranch() { + assumeThat(formatVersion) + .isGreaterThanOrEqualTo(TableMetadata.MIN_FORMAT_VERSION_MONOTONIC_TIMESTAMPS); + + // Establish a base snapshot on main. + table.newFastAppend().appendFile(FILE_A).commit(); + long mainParentTs = table.currentSnapshot().timestampMillis(); + + // Create a branch pointing at the current main head. + String branchName = "test-branch"; + table.manageSnapshots().createBranch(branchName).commit(); + + // Commit to the branch with a clock far in the future so the branch head's + // timestamp-ms is much higher than main's head timestamp-ms. + long branchFutureTs = mainParentTs + 1_000_000L; + Clock branchClock = Clock.fixed(Instant.ofEpochMilli(branchFutureTs), ZoneOffset.UTC); + AppendFiles branchAppend = table.newFastAppend().appendFile(FILE_B).toBranch(branchName); + ((SnapshotProducer) branchAppend).setClock(branchClock); + branchAppend.commit(); + long branchTs = table.snapshot(branchName).timestampMillis(); + assertThat(branchTs) + .as("Sanity: branch commit should adopt the simulated far-future wall-clock value") + .isEqualTo(branchFutureTs); + + // Commit to main with a clock just slightly after mainParentTs but far below branchTs. + // The spec requires monotonicity "on the same branch", so main's new timestamp must be + // constrained only by main's previous head, not by the unrelated future timestamp on + // the other branch. + long mainNewTs = mainParentTs + 5; + Clock mainClock = Clock.fixed(Instant.ofEpochMilli(mainNewTs), ZoneOffset.UTC); + AppendFiles mainAppend = table.newFastAppend().appendFile(FILE_C); + ((SnapshotProducer) mainAppend).setClock(mainClock); + mainAppend.commit(); + + long actualMainTs = table.currentSnapshot().timestampMillis(); + assertThat(actualMainTs) + .as("Main's monotonicity must be relative to main's parent, not the branch head") + .isEqualTo(mainNewTs) + .isLessThan(branchTs); + } } From cead2eaf69932b67402794ac9cb9a1a1c9773035 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 28 Sep 2026 15:41:24 -0700 Subject: [PATCH 3/5] Core: produce Lamport snapshot timestamps for all format versions Always fast-forward snapshot timestamp-ms past the parent so Java writers stay monotonic going forward. Validate parent-to-child timestamps only when adding v4 snapshots, and stop comparing last-updated-ms to snapshot-log times that can now be logically ahead of wall clock. Generated-by: Cursor Grok 4.6 Co-authored-by: Cursor --- .../org/apache/iceberg/SnapshotProducer.java | 13 +-- .../org/apache/iceberg/TableMetadata.java | 21 ++--- .../apache/iceberg/TestSnapshotProducer.java | 80 +++++++++++++------ .../org/apache/iceberg/TestTableMetadata.java | 57 +++++++++++++ 4 files changed, 125 insertions(+), 46 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/SnapshotProducer.java b/core/src/main/java/org/apache/iceberg/SnapshotProducer.java index 63d97f3b15ec..daeded0ee00a 100644 --- a/core/src/main/java/org/apache/iceberg/SnapshotProducer.java +++ b/core/src/main/java/org/apache/iceberg/SnapshotProducer.java @@ -690,20 +690,9 @@ void setClock(Clock newClock) { this.clock = newClock; } - /** - * Generates the snapshot timestamp in milliseconds. - * - *

For format version 4 and above, this implements the Lamport clock algorithm to guarantee - * monotonically increasing snapshot timestamps. For older format versions, this returns the - * current wall clock time. - * - * @param parentSnapshot the parent snapshot on the target branch, or null if there is no parent - * @return the snapshot timestamp in milliseconds - */ private long snapshotTimestampMillis(Snapshot parentSnapshot) { long now = clock.millis(); - if (base.formatVersion() >= TableMetadata.MIN_FORMAT_VERSION_MONOTONIC_TIMESTAMPS - && parentSnapshot != null) { + if (parentSnapshot != null) { return Math.max(now, parentSnapshot.timestampMillis() + 1); } diff --git a/core/src/main/java/org/apache/iceberg/TableMetadata.java b/core/src/main/java/org/apache/iceberg/TableMetadata.java index 1fdc2ce70cf1..20884ed7e32a 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadata.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadata.java @@ -379,15 +379,6 @@ public String toString() { } last = logEntry; } - if (last != null) { - Preconditions.checkArgument( - // commits can happen concurrently from different machines. - // A tolerance helps us avoid failure for small clock skew - lastUpdatedMillis - last.timestampMillis() >= -ONE_MINUTE, - "Invalid update timestamp %s: before last snapshot log entry at %s", - lastUpdatedMillis, - last.timestampMillis()); - } MetadataLogEntry previous = null; for (MetadataLogEntry metadataEntry : previousFiles) { @@ -1271,6 +1262,18 @@ public Builder addSnapshot(Snapshot snapshot) { snapshot.sequenceNumber(), lastSequenceNumber); + if (formatVersion >= MIN_FORMAT_VERSION_MONOTONIC_TIMESTAMPS && snapshot.parentId() != null) { + Snapshot parent = snapshotsById.get(snapshot.parentId()); + if (parent != null) { + ValidationException.check( + snapshot.timestampMillis() > parent.timestampMillis(), + "Invalid snapshot timestamp %s: not after parent snapshot %s at %s", + snapshot.timestampMillis(), + snapshot.parentId(), + parent.timestampMillis()); + } + } + this.lastSequenceNumber = snapshot.sequenceNumber(); snapshots.add(snapshot); snapshotsById.put(snapshot.snapshotId(), snapshot); diff --git a/core/src/test/java/org/apache/iceberg/TestSnapshotProducer.java b/core/src/test/java/org/apache/iceberg/TestSnapshotProducer.java index 802d3ed6c386..a01411b66c5b 100644 --- a/core/src/test/java/org/apache/iceberg/TestSnapshotProducer.java +++ b/core/src/test/java/org/apache/iceberg/TestSnapshotProducer.java @@ -33,6 +33,7 @@ import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import javax.annotation.Nonnull; import org.apache.iceberg.exceptions.ValidationException; @@ -281,54 +282,83 @@ public void testWriteManifestsWithInvalidParallelismThrows() { } @TestTemplate - public void testSnapshotTimestampsAreMonotonicallyIncreasing() { - assumeThat(formatVersion) - .isGreaterThanOrEqualTo(TableMetadata.MIN_FORMAT_VERSION_MONOTONIC_TIMESTAMPS); - + public void snapshotTimestampsAreMonotonicallyIncreasing() { table.newFastAppend().appendFile(FILE_A).commit(); Snapshot first = table.currentSnapshot(); table.newFastAppend().appendFile(FILE_B).commit(); Snapshot second = table.currentSnapshot(); - assertThat(second.timestampMillis()) - .as("V4 snapshot timestamps must be strictly increasing") - .isGreaterThan(first.timestampMillis()); + assertThat(second.timestampMillis()).isGreaterThan(first.timestampMillis()); table.newFastAppend().appendFile(FILE_C).commit(); Snapshot third = table.currentSnapshot(); - assertThat(third.timestampMillis()) - .as("V4 snapshot timestamps must be strictly increasing") - .isGreaterThan(second.timestampMillis()); + assertThat(third.timestampMillis()).isGreaterThan(second.timestampMillis()); } @TestTemplate - public void testV4LamportClockFastForwardsDriftedClock() { - assumeThat(formatVersion) - .isGreaterThanOrEqualTo(TableMetadata.MIN_FORMAT_VERSION_MONOTONIC_TIMESTAMPS); - + public void lamportClockFastForwardsDriftedClock() { table.newFastAppend().appendFile(FILE_A).commit(); long firstTs = table.currentSnapshot().timestampMillis(); - // simulate clock drift: wall clock reports a time far in the past - long driftedTime = firstTs - 10_000; + long driftedTime = firstTs - TimeUnit.MINUTES.toMillis(2); Clock driftedClock = Clock.fixed(Instant.ofEpochMilli(driftedTime), ZoneOffset.UTC); AppendFiles append = table.newFastAppend().appendFile(FILE_B); ((SnapshotProducer) append).setClock(driftedClock); append.commit(); - long secondTs = table.currentSnapshot().timestampMillis(); - assertThat(secondTs) - .as( - "Lamport clock should fast-forward past the drifted wall clock to the last snapshot timestamp + 1 ms") - .isEqualTo(firstTs + 1); + assertThat(table.currentSnapshot().timestampMillis()).isEqualTo(firstTs + 1); } @TestTemplate - public void testV4MonotonicityIsScopedToTargetBranch() { - assumeThat(formatVersion) - .isGreaterThanOrEqualTo(TableMetadata.MIN_FORMAT_VERSION_MONOTONIC_TIMESTAMPS); + public void transactionCommitsProduceMonotonicTimestamps() { + Transaction txn = table.newTransaction(); + txn.newFastAppend().appendFile(FILE_A).commit(); + Snapshot first = txn.table().currentSnapshot(); + txn.newFastAppend().appendFile(FILE_B).commit(); + Snapshot second = txn.table().currentSnapshot(); + txn.commitTransaction(); + + assertThat(second.parentId()).isEqualTo(first.snapshotId()); + assertThat(second.timestampMillis()).isGreaterThan(first.timestampMillis()); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(second.snapshotId()); + } - // Establish a base snapshot on main. + @TestTemplate + public void transactionRetryFastForwardsTimestampsPastConflictingSnapshot() { + table.updateProperties().set(TableProperties.COMMIT_NUM_RETRIES, "1").commit(); + + table.newFastAppend().appendFile(FILE_A).commit(); + long baseTs = table.currentSnapshot().timestampMillis(); + + Clock staleClock = Clock.fixed(Instant.ofEpochMilli(baseTs + 1), ZoneOffset.UTC); + Transaction txn = table.newTransaction(); + AppendFiles firstAppend = txn.newFastAppend().appendFile(FILE_B); + ((SnapshotProducer) firstAppend).setClock(staleClock); + firstAppend.commit(); + AppendFiles secondAppend = txn.newFastAppend().appendFile(FILE_C); + ((SnapshotProducer) secondAppend).setClock(staleClock); + secondAppend.commit(); + + long conflictTs = baseTs + 1_000_000L; + AppendFiles conflictAppend = table.newFastAppend().appendFile(FILE_D); + ((SnapshotProducer) conflictAppend) + .setClock(Clock.fixed(Instant.ofEpochMilli(conflictTs), ZoneOffset.UTC)); + conflictAppend.commit(); + Snapshot conflict = table.currentSnapshot(); + assertThat(conflict.timestampMillis()).isEqualTo(conflictTs); + assertThat(txn.table().currentSnapshot().timestampMillis()).isLessThan(conflictTs); + + txn.commitTransaction(); + + Snapshot retriedSecond = table.currentSnapshot(); + Snapshot retriedFirst = table.snapshot(retriedSecond.parentId()); + assertThat(retriedFirst.parentId()).isEqualTo(conflict.snapshotId()); + assertThat(retriedFirst.timestampMillis()).isGreaterThan(conflict.timestampMillis()); + assertThat(retriedSecond.timestampMillis()).isGreaterThan(retriedFirst.timestampMillis()); + } + + @TestTemplate + public void monotonicityIsScopedToTargetBranch() { table.newFastAppend().appendFile(FILE_A).commit(); long mainParentTs = table.currentSnapshot().timestampMillis(); diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java index ce176dd2bee9..468b260edfce 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java @@ -52,6 +52,7 @@ import java.util.Set; import java.util.SortedSet; import java.util.UUID; +import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; import java.util.stream.Stream; import org.apache.iceberg.TableMetadata.MetadataLogEntry; @@ -2186,4 +2187,60 @@ public void testSetRefRejectsTagForMainBranch() { .build(); assertThat(withTag.ref("tag1").isTag()).isTrue(); } + + @Test + public void addSnapshotRejectsNonMonotonicTimestampForV4() { + TableMetadata base = + TableMetadata.newTableMetadata( + TEST_SCHEMA, + PartitionSpec.unpartitioned(), + TEST_LOCATION, + ImmutableMap.of(TableProperties.FORMAT_VERSION, "4")); + + Snapshot parent = + new BaseSnapshot(1, 1L, null, 1_000L, null, null, null, "file:/s1.avro", 0L, 0L, null); + TableMetadata withParent = TableMetadata.buildFrom(base).addSnapshot(parent).build(); + + Snapshot child = + new BaseSnapshot(2, 2L, 1L, 1_000L, null, null, null, "file:/s2.avro", 0L, 0L, null); + + assertThatThrownBy(() -> TableMetadata.buildFrom(withParent).addSnapshot(child)) + .isInstanceOf(ValidationException.class) + .hasMessage("Invalid snapshot timestamp 1000: not after parent snapshot 1 at 1000"); + } + + @Test + public void addSnapshotAllowsNonMonotonicTimestampBeforeV4() { + TableMetadata base = + TableMetadata.newTableMetadata( + TEST_SCHEMA, PartitionSpec.unpartitioned(), "location", ImmutableMap.of()); + + Snapshot parent = + new BaseSnapshot(1, 1L, null, 1_000L, null, null, null, "file:/s1.avro", null, null, null); + TableMetadata withParent = TableMetadata.buildFrom(base).addSnapshot(parent).build(); + + Snapshot child = + new BaseSnapshot(2, 2L, 1L, 999L, null, null, null, "file:/s2.avro", null, null, null); + + TableMetadata withChild = TableMetadata.buildFrom(withParent).addSnapshot(child).build(); + assertThat(withChild.snapshot(2L).timestampMillis()).isEqualTo(999L); + } + + @Test + public void snapshotTimestampMayBeAheadOfLastUpdatedMillis() { + TableMetadata base = + TableMetadata.newTableMetadata( + TEST_SCHEMA, PartitionSpec.unpartitioned(), TEST_LOCATION, ImmutableMap.of()); + + long snapshotTs = System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(2); + Snapshot snapshot = + new BaseSnapshot( + 1, 1L, null, snapshotTs, null, null, null, "file:/s1.avro", null, null, null); + + TableMetadata updated = + TableMetadata.buildFrom(base).setBranchSnapshot(snapshot, SnapshotRef.MAIN_BRANCH).build(); + + assertThat(updated.currentSnapshot().timestampMillis()).isEqualTo(snapshotTs); + assertThat(updated.lastUpdatedMillis()).isLessThan(snapshotTs); + } } From c45490674f8a403ee192a33f87721cd522fd1748 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 28 Sep 2026 16:05:08 -0700 Subject: [PATCH 4/5] Core: give v4 row-lineage test snapshots increasing timestamps v4 addSnapshot now requires timestamp-ms strictly after the parent, so these fixtures cannot reuse 0 for both snapshots. Generated-by: Cursor Grok 4.6 Co-authored-by: Cursor --- .../test/java/org/apache/iceberg/TestRowLineageMetadata.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/org/apache/iceberg/TestRowLineageMetadata.java b/core/src/test/java/org/apache/iceberg/TestRowLineageMetadata.java index 4fd4f9d3614e..4d861d5a815c 100644 --- a/core/src/test/java/org/apache/iceberg/TestRowLineageMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestRowLineageMetadata.java @@ -127,7 +127,7 @@ public void testSnapshotAddition() { 1, 2, 1L, - 0, + 1, DataOperations.APPEND, null, 1, @@ -163,7 +163,7 @@ public void testInvalidSnapshotAddition() { Snapshot invalidNewRows = new BaseSnapshot( - 1, 2, 1L, 0, DataOperations.APPEND, null, 1, "foo", added.nextRowId() - 1, 10L, null); + 1, 2, 1L, 1, DataOperations.APPEND, null, 1, "foo", added.nextRowId() - 1, 10L, null); assertThatThrownBy(() -> TableMetadata.buildFrom(added).addSnapshot(invalidNewRows)) .isInstanceOf(ValidationException.class) From c12134e93a380204b2c8f60c036a3e690f1fcc0e Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Wed, 7 Oct 2026 12:26:27 -0700 Subject: [PATCH 5/5] Core: align monotonic timestamp tests with format versions Parameterize metadata checks by version lists and cover transaction commits and retry clocks more realistically. Generated-by: Cursor Grok 4.7 Co-authored-by: Cursor --- .../java/org/apache/iceberg/TestHelpers.java | 4 + .../apache/iceberg/TestSnapshotProducer.java | 25 ++++-- .../org/apache/iceberg/TestTableMetadata.java | 83 ++++++++++++++++--- 3 files changed, 91 insertions(+), 21 deletions(-) diff --git a/api/src/test/java/org/apache/iceberg/TestHelpers.java b/api/src/test/java/org/apache/iceberg/TestHelpers.java index 58d992d8df3e..ee9c169aa814 100644 --- a/api/src/test/java/org/apache/iceberg/TestHelpers.java +++ b/api/src/test/java/org/apache/iceberg/TestHelpers.java @@ -80,6 +80,10 @@ private TestHelpers() {} IntStream.rangeClosed(2, MAX_FORMAT_VERSION).boxed().collect(Collectors.toUnmodifiableList()); public static final List V3_AND_ABOVE = IntStream.rangeClosed(3, MAX_FORMAT_VERSION).boxed().collect(Collectors.toUnmodifiableList()); + public static final List V4_AND_ABOVE = + IntStream.rangeClosed(4, MAX_FORMAT_VERSION).boxed().collect(Collectors.toUnmodifiableList()); + public static final List V3_AND_BELOW = + IntStream.rangeClosed(1, 3).boxed().collect(Collectors.toUnmodifiableList()); /** Wait in a tight check loop until system clock is past {@code timestampMillis} */ public static long waitUntilAfter(long timestampMillis) { diff --git a/core/src/test/java/org/apache/iceberg/TestSnapshotProducer.java b/core/src/test/java/org/apache/iceberg/TestSnapshotProducer.java index a01411b66c5b..8d128b9b6b4f 100644 --- a/core/src/test/java/org/apache/iceberg/TestSnapshotProducer.java +++ b/core/src/test/java/org/apache/iceberg/TestSnapshotProducer.java @@ -306,18 +306,23 @@ public void lamportClockFastForwardsDriftedClock() { ((SnapshotProducer) append).setClock(driftedClock); append.commit(); - assertThat(table.currentSnapshot().timestampMillis()).isEqualTo(firstTs + 1); + assertThat(table.currentSnapshot().timestampMillis()).isGreaterThan(firstTs); } @TestTemplate public void transactionCommitsProduceMonotonicTimestamps() { + table.newFastAppend().appendFile(FILE_A).commit(); + Snapshot base = table.currentSnapshot(); + Transaction txn = table.newTransaction(); - txn.newFastAppend().appendFile(FILE_A).commit(); - Snapshot first = txn.table().currentSnapshot(); txn.newFastAppend().appendFile(FILE_B).commit(); + Snapshot first = txn.table().currentSnapshot(); + txn.newFastAppend().appendFile(FILE_C).commit(); Snapshot second = txn.table().currentSnapshot(); txn.commitTransaction(); + assertThat(first.parentId()).isEqualTo(base.snapshotId()); + assertThat(first.timestampMillis()).isGreaterThan(base.timestampMillis()); assertThat(second.parentId()).isEqualTo(first.snapshotId()); assertThat(second.timestampMillis()).isGreaterThan(first.timestampMillis()); assertThat(table.currentSnapshot().snapshotId()).isEqualTo(second.snapshotId()); @@ -329,14 +334,18 @@ public void transactionRetryFastForwardsTimestampsPastConflictingSnapshot() { table.newFastAppend().appendFile(FILE_A).commit(); long baseTs = table.currentSnapshot().timestampMillis(); - - Clock staleClock = Clock.fixed(Instant.ofEpochMilli(baseTs + 1), ZoneOffset.UTC); Transaction txn = table.newTransaction(); + + long firstCommitTs = baseTs + 1; + Clock firstClock = Clock.fixed(Instant.ofEpochMilli(firstCommitTs), ZoneOffset.UTC); AppendFiles firstAppend = txn.newFastAppend().appendFile(FILE_B); - ((SnapshotProducer) firstAppend).setClock(staleClock); + ((SnapshotProducer) firstAppend).setClock(firstClock); firstAppend.commit(); + + long secondCommitTs = firstCommitTs + 1; + Clock secondClock = Clock.fixed(Instant.ofEpochMilli(secondCommitTs), ZoneOffset.UTC); AppendFiles secondAppend = txn.newFastAppend().appendFile(FILE_C); - ((SnapshotProducer) secondAppend).setClock(staleClock); + ((SnapshotProducer) secondAppend).setClock(secondClock); secondAppend.commit(); long conflictTs = baseTs + 1_000_000L; @@ -375,7 +384,7 @@ public void monotonicityIsScopedToTargetBranch() { branchAppend.commit(); long branchTs = table.snapshot(branchName).timestampMillis(); assertThat(branchTs) - .as("Sanity: branch commit should adopt the simulated far-future wall-clock value") + .as("branch commit should adopt the simulated far-future wall-clock value") .isEqualTo(branchFutureTs); // Commit to main with a clock just slightly after mainParentTs but far below branchTs. diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java index 468b260edfce..f4a6d795d77f 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java @@ -73,6 +73,7 @@ import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.FieldSource; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; @@ -2188,14 +2189,15 @@ public void testSetRefRejectsTagForMainBranch() { assertThat(withTag.ref("tag1").isTag()).isTrue(); } - @Test - public void addSnapshotRejectsNonMonotonicTimestampForV4() { + @ParameterizedTest + @FieldSource("org.apache.iceberg.TestHelpers#V4_AND_ABOVE") + public void addSnapshotRejectsNonMonotonicTimestampAfterV4(int formatVersion) { TableMetadata base = TableMetadata.newTableMetadata( TEST_SCHEMA, PartitionSpec.unpartitioned(), TEST_LOCATION, - ImmutableMap.of(TableProperties.FORMAT_VERSION, "4")); + ImmutableMap.of(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion))); Snapshot parent = new BaseSnapshot(1, 1L, null, 1_000L, null, null, null, "file:/s1.avro", 0L, 0L, null); @@ -2209,33 +2211,88 @@ public void addSnapshotRejectsNonMonotonicTimestampForV4() { .hasMessage("Invalid snapshot timestamp 1000: not after parent snapshot 1 at 1000"); } - @Test - public void addSnapshotAllowsNonMonotonicTimestampBeforeV4() { + @ParameterizedTest + @FieldSource("org.apache.iceberg.TestHelpers#V3_AND_BELOW") + public void addSnapshotAllowsNonMonotonicTimestampBeforeV4(int formatVersion) { TableMetadata base = TableMetadata.newTableMetadata( - TEST_SCHEMA, PartitionSpec.unpartitioned(), "location", ImmutableMap.of()); - + TEST_SCHEMA, + PartitionSpec.unpartitioned(), + "location", + ImmutableMap.of(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion))); + + // v1-v3 differ in sequence number and row-lineage fields. + long parentSequenceNumber = formatVersion == 1 ? 0L : 1L; + Long parentFirstRowId = + formatVersion >= TableMetadata.MIN_FORMAT_VERSION_ROW_LINEAGE ? 0L : null; + Long parentAddedRows = + formatVersion >= TableMetadata.MIN_FORMAT_VERSION_ROW_LINEAGE ? 10L : null; Snapshot parent = - new BaseSnapshot(1, 1L, null, 1_000L, null, null, null, "file:/s1.avro", null, null, null); + new BaseSnapshot( + parentSequenceNumber, + 1L, + null, + 1_000L, + null, + null, + null, + "file:/s1.avro", + parentFirstRowId, + parentAddedRows, + null); TableMetadata withParent = TableMetadata.buildFrom(base).addSnapshot(parent).build(); + long childSequenceNumber = formatVersion == 1 ? 0L : 2L; + Long childFirstRowId = + formatVersion >= TableMetadata.MIN_FORMAT_VERSION_ROW_LINEAGE ? parentAddedRows : null; + Long childAddedRows = + formatVersion >= TableMetadata.MIN_FORMAT_VERSION_ROW_LINEAGE ? 20L : null; Snapshot child = - new BaseSnapshot(2, 2L, 1L, 999L, null, null, null, "file:/s2.avro", null, null, null); + new BaseSnapshot( + childSequenceNumber, + 2L, + 1L, + 999L, + null, + null, + null, + "file:/s2.avro", + childFirstRowId, + childAddedRows, + null); TableMetadata withChild = TableMetadata.buildFrom(withParent).addSnapshot(child).build(); assertThat(withChild.snapshot(2L).timestampMillis()).isEqualTo(999L); } - @Test - public void snapshotTimestampMayBeAheadOfLastUpdatedMillis() { + @ParameterizedTest + @FieldSource("org.apache.iceberg.TestHelpers#ALL_VERSIONS") + public void snapshotTimestampMayBeAheadOfTableMetadataLastUpdatedMillis(int formatVersion) { TableMetadata base = TableMetadata.newTableMetadata( - TEST_SCHEMA, PartitionSpec.unpartitioned(), TEST_LOCATION, ImmutableMap.of()); + TEST_SCHEMA, + PartitionSpec.unpartitioned(), + TEST_LOCATION, + ImmutableMap.of(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion))); long snapshotTs = System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(2); + // Versions differ in sequence number and row-lineage fields. + long sequenceNumber = formatVersion == 1 ? 0L : 1L; + Long firstRowId = formatVersion >= TableMetadata.MIN_FORMAT_VERSION_ROW_LINEAGE ? 0L : null; + Long addedRows = formatVersion >= TableMetadata.MIN_FORMAT_VERSION_ROW_LINEAGE ? 10L : null; Snapshot snapshot = new BaseSnapshot( - 1, 1L, null, snapshotTs, null, null, null, "file:/s1.avro", null, null, null); + sequenceNumber, + 1L, + null, + snapshotTs, + null, + null, + null, + "file:/s1.avro", + firstRowId, + addedRows, + null); TableMetadata updated = TableMetadata.buildFrom(base).setBranchSnapshot(snapshot, SnapshotRef.MAIN_BRANCH).build();