From 0a7f8ffc2d7ee4bb6c37d05853fc6bdf1f57f011 Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Tue, 17 Nov 2026 23:08:49 +0000 Subject: [PATCH 1/2] Row lineage: Read _last_updated_sequence_number Carry the data sequence number of each data file on FileScanTask, and fill null _last_updated_sequence_number values with it, as the Java readers do. The value is null when the file has no first row ID or no data sequence number, such as for files planned by a REST server. Part of #4080. --- pyiceberg/io/pyarrow.py | 32 ++++++++++++-- pyiceberg/table/__init__.py | 4 ++ tests/integration/test_reads.py | 21 +++++---- tests/io/test_pyarrow.py | 75 ++++++++++++++++++++++++++++++--- 4 files changed, 112 insertions(+), 20 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 5961da883c..67d7360729 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -1801,6 +1801,14 @@ def _task_to_record_batches( (index for index, field in enumerate(projected_schema.fields) if field.field_id == ROW_ID.field_id), None ) filter_after_read = bool(positional_deletes) or row_id_index is not None + last_updated_index = next( + ( + index + for index, field in enumerate(projected_schema.fields) + if field.field_id == LAST_UPDATED_SEQUENCE_NUMBER.field_id + ), + None, + ) fragment_scanner = ds.Scanner.from_fragment( fragment=fragment, @@ -1856,6 +1864,10 @@ def _task_to_record_batches( ) if row_id_index is not None and positions is not None: result_batch = _assign_row_ids(result_batch, row_id_index, positions, task.file.first_row_id) + if last_updated_index is not None: + result_batch = _assign_last_updated_sequence_numbers( + result_batch, last_updated_index, task.file.first_row_id, task.data_sequence_number + ) yield result_batch @@ -1871,6 +1883,23 @@ def _assign_row_ids(batch: pa.RecordBatch, row_id_index: int, positions: pa.Arra return batch.set_column(row_id_index, batch.schema.field(row_id_index), row_ids) +def _assign_last_updated_sequence_numbers( + batch: pa.RecordBatch, last_updated_index: int, first_row_id: int | None, data_sequence_number: int | None +) -> pa.RecordBatch: + """Replace null last updated sequence numbers with the data sequence number of the file. + + When the data file has no first row ID, every last updated sequence number is null. + """ + if first_row_id is None: + sequence_numbers = pa.nulls(len(batch), type=pa.int64()) + elif data_sequence_number is None: + # REST scan planning does not return the data sequence number + raise NotImplementedError(f"Reading {LAST_UPDATED_SEQUENCE_NUMBER.name} requires the data sequence number of the file") + else: + sequence_numbers = pc.coalesce(batch.column(last_updated_index), pa.scalar(data_sequence_number, type=pa.int64())) + return batch.set_column(last_updated_index, batch.schema.field(last_updated_index), sequence_numbers) + + @dataclass(frozen=True, slots=True) class _DeleteFileKey: """Identity of a delete file or a referenced DV range.""" @@ -1937,9 +1966,6 @@ def __init__( *, dictionary_columns: tuple[str, ...] = (), ) -> None: - if any(field.field_id == LAST_UPDATED_SEQUENCE_NUMBER.field_id for field in projected_schema.fields): - # The data sequence number is not carried on the scan task yet - raise NotImplementedError(f"Reading {LAST_UPDATED_SEQUENCE_NUMBER.name} is not yet supported") self._table_metadata = table_metadata self._io = io self._projected_schema = projected_schema diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 4d65084cbf..de6f9e4b6b 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -2328,16 +2328,19 @@ class FileScanTask(ScanTask): file: DataFile delete_files: set[DataFile] residual: BooleanExpression + data_sequence_number: int | None def __init__( self, data_file: DataFile, delete_files: set[DataFile] | None = None, residual: BooleanExpression = ALWAYS_TRUE, + data_sequence_number: int | None = None, ) -> None: self.file = data_file self.delete_files = delete_files or set() self.residual = residual + self.data_sequence_number = data_sequence_number @staticmethod def from_rest_response( @@ -2996,6 +2999,7 @@ def residual_for(data_file: DataFile) -> BooleanExpression: partition_key=data_entry.data_file.partition, ), residual=ALWAYS_TRUE if self._ignore_residuals else residual_for(data_entry.data_file), + data_sequence_number=data_entry.sequence_number, ) for data_entry in data_entries ] diff --git a/tests/integration/test_reads.py b/tests/integration/test_reads.py index f3e55f48e6..6a524140a2 100644 --- a/tests/integration/test_reads.py +++ b/tests/integration/test_reads.py @@ -1301,8 +1301,8 @@ def test_read_first_row_ids_written_by_spark(catalog: Catalog, spark: SparkSessi @pytest.mark.integration @pytest.mark.parametrize("catalog", [lf("session_catalog")]) -def test_read_row_ids_written_by_spark(catalog: Catalog, spark: SparkSession) -> None: - identifier = "default.test_read_row_ids_written_by_spark" +def test_read_row_lineage_written_by_spark(catalog: Catalog, spark: SparkSession) -> None: + identifier = "default.test_read_row_lineage_written_by_spark" spark.sql(f"DROP TABLE IF EXISTS {identifier}") spark.sql( f""" @@ -1335,17 +1335,16 @@ def test_read_row_ids_written_by_spark(catalog: Catalog, spark: SparkSession) -> # Rows in a new file inherit row IDs from their position, so a filter must not drop rows before they are counted spark.sql(f"INSERT INTO {identifier} SELECT /*+ COALESCE(1) */ * FROM VALUES (8, 'h'), (9, 'i'), (10, 'j')") - expected = sorted((row.id, row._row_id) for row in spark.sql(f"SELECT id, _row_id FROM {identifier}").collect()) - assert all(row_id is not None for _, row_id in expected) + columns = ("id", "_row_id", "_last_updated_sequence_number") + expected = sorted(tuple(row) for row in spark.sql(f"SELECT {', '.join(columns)} FROM {identifier}").collect()) + assert all(value is not None for row in expected for value in row) - table = catalog.load_table(identifier) - result = table.scan(selected_fields=("id", "_row_id")).to_arrow() - assert sorted(zip(result["id"].to_pylist(), result["_row_id"].to_pylist(), strict=True)) == expected + def rows(result: pa.Table) -> list[tuple[int, int, int]]: + return sorted(zip(*(result[column].to_pylist() for column in columns), strict=True)) - filtered = table.scan(row_filter="id >= 9", selected_fields=("id", "_row_id")).to_arrow() - assert sorted(zip(filtered["id"].to_pylist(), filtered["_row_id"].to_pylist(), strict=True)) == [ - (id_, row_id) for id_, row_id in expected if id_ >= 9 - ] + table = catalog.load_table(identifier) + assert rows(table.scan(selected_fields=columns).to_arrow()) == expected + assert rows(table.scan(row_filter="id >= 9", selected_fields=columns).to_arrow()) == [row for row in expected if row[0] >= 9] @pytest.mark.integration diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index b0f1cdde37..161058536c 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -1344,12 +1344,6 @@ def test_read_map(schema_map: Schema, file_map: str) -> None: ) -def test_projection_last_updated_sequence_number_not_supported(file_int: str) -> None: - schema = Schema(NestedField(1, "id", IntegerType(), required=False), LAST_UPDATED_SEQUENCE_NUMBER) - with pytest.raises(NotImplementedError, match="Reading _last_updated_sequence_number is not yet supported"): - project(schema, [file_int]) - - def test_projection_add_column_struct(schema_int: Schema, file_int: str) -> None: schema = Schema( # A new ID @@ -4501,6 +4495,75 @@ def test_task_to_record_batches_row_ids( assert list(zip(result["id"].to_pylist(), result[ROW_ID.name].to_pylist(), strict=True)) == expected +def _read_last_updated_sequence_numbers( + tmpdir: str, stored_sequence_numbers: list[int | None] | None, first_row_id: int | None, data_sequence_number: int | None +) -> list[int | None]: + fields = [pa.field("id", pa.int32(), nullable=True, metadata={PYARROW_PARQUET_FIELD_ID_KEY: "1"})] + columns = [pa.array([1, 2, 3], type=pa.int32())] + if stored_sequence_numbers is not None: + fields.append( + pa.field( + LAST_UPDATED_SEQUENCE_NUMBER.name, + pa.int64(), + nullable=True, + metadata={PYARROW_PARQUET_FIELD_ID_KEY: str(LAST_UPDATED_SEQUENCE_NUMBER.field_id)}, + ) + ) + columns.append(pa.array(stored_sequence_numbers, type=pa.int64())) + arrow_schema = pa.schema(fields) + data_file = _write_table_to_data_file( + f"{tmpdir}/last_updated_sequence_numbers.parquet", arrow_schema, pa.table(columns, schema=arrow_schema) + ) + data_file.first_row_id = first_row_id + + table_schema = Schema(NestedField(1, "id", IntegerType(), required=False)) + projected_schema = Schema(*table_schema.fields, ROW_ID, LAST_UPDATED_SEQUENCE_NUMBER) + result_batches = list( + _task_to_record_batches( + PyArrowFileIO(), + FileScanTask(data_file, data_sequence_number=data_sequence_number), + bound_row_filter=AlwaysTrue(), + projected_schema=projected_schema, + table_schema=table_schema, + projected_field_ids={1, ROW_ID.field_id, LAST_UPDATED_SEQUENCE_NUMBER.field_id}, + positional_deletes=None, + case_sensitive=True, + ) + ) + + result = pa.Table.from_batches(result_batches) + assert result.schema.field(LAST_UPDATED_SEQUENCE_NUMBER.name).type == pa.int64() + return result[LAST_UPDATED_SEQUENCE_NUMBER.name].to_pylist() + + +@pytest.mark.parametrize( + "stored_sequence_numbers, first_row_id, data_sequence_number, expected", + [ + # Last updated sequence numbers are inherited from the data sequence number of the file + (None, 100, 5, [5, 5, 5]), + # Last updated sequence numbers stored in the file take precedence + ([None, 2, None], 100, 5, [5, 2, 5]), + # Without a first row ID, last updated sequence numbers are null even when stored in the file + ([None, 2, None], None, 5, [None, None, None]), + # Without a first row ID, the data sequence number is not needed + ([None, 2, None], None, None, [None, None, None]), + ], +) +def test_task_to_record_batches_last_updated_sequence_numbers( + tmpdir: str, + stored_sequence_numbers: list[int | None] | None, + first_row_id: int | None, + data_sequence_number: int | None, + expected: list[int | None], +) -> None: + assert _read_last_updated_sequence_numbers(tmpdir, stored_sequence_numbers, first_row_id, data_sequence_number) == expected + + +def test_task_to_record_batches_last_updated_sequence_numbers_without_data_sequence_number(tmpdir: str) -> None: + with pytest.raises(NotImplementedError, match="requires the data sequence number"): + _read_last_updated_sequence_numbers(tmpdir, [None, 2, None], first_row_id=100, data_sequence_number=None) + + def test_task_to_record_batches_encrypted_by_iceberg_rust() -> None: """The data file was encrypted by iceberg-rust, see tests/table/encryption/README.md.""" data_file = DataFile.from_args( From 0cdeab5c929310aea584e4b30d33878b1aa97994 Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Tue, 17 Nov 2026 23:11:42 +0000 Subject: [PATCH 2/2] Row lineage: Show first_row_id in inspect.files() Add the first_row_id column to the files metadata tables, matching the Java files table schema. Part of #4080. --- mkdocs/docs/api.md | 2 ++ pyiceberg/table/inspect.py | 4 +++- tests/integration/test_inspect_table.py | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index a732d2c0a3..86665072e7 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -985,6 +985,7 @@ split_offsets: list equality_ids: list child 0, item: int32 sort_order_id: int32 +first_row_id: int64 readable_metrics: struct not null, lat: struct not null, long: struct not null> child 0, city: struct not null child 0, column_size: int64 @@ -1024,6 +1025,7 @@ key_metadata: [[0100,0100]] split_offsets:[[[],[]]] equality_ids:[[[],[]]] sort_order_id:[[[],[]]] +first_row_id:[[null,null]] readable_metrics: [ -- is_valid: all not null -- child 0 type: struct diff --git a/pyiceberg/table/inspect.py b/pyiceberg/table/inspect.py index d51079fc9d..6ccc6a500d 100644 --- a/pyiceberg/table/inspect.py +++ b/pyiceberg/table/inspect.py @@ -784,6 +784,7 @@ def _get_files_from_manifest( "split_offsets": data_file.split_offsets, "equality_ids": data_file.equality_ids, "sort_order_id": data_file.sort_order_id, + "first_row_id": data_file.first_row_id, "readable_metrics": readable_metrics, } ) @@ -804,7 +805,7 @@ def _get_files_schema(self) -> pa.Schema: ``file_size_in_bytes``, ``column_sizes``, ``value_counts``, ``null_value_counts``, ``nan_value_counts``, ``lower_bounds``, ``upper_bounds``, ``key_metadata``, ``split_offsets``, - ``equality_ids``, ``sort_order_id``, and ``readable_metrics``. + ``equality_ids``, ``sort_order_id``, ``first_row_id``, and ``readable_metrics``. """ import pyarrow as pa @@ -853,6 +854,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType: pa.field("split_offsets", pa.list_(pa.int64()), nullable=True), pa.field("equality_ids", pa.list_(pa.int32()), nullable=True), pa.field("sort_order_id", pa.int32(), nullable=True), + pa.field("first_row_id", pa.int64(), nullable=True), pa.field("readable_metrics", pa.struct(readable_metrics_struct), nullable=True), ] ) diff --git a/tests/integration/test_inspect_table.py b/tests/integration/test_inspect_table.py index 4d8dfbe9bb..7868bb0f25 100644 --- a/tests/integration/test_inspect_table.py +++ b/tests/integration/test_inspect_table.py @@ -101,6 +101,7 @@ def _inspect_files_asserts(df: pa.Table, spark_df: DataFrame) -> None: "split_offsets", "equality_ids", "sort_order_id", + "first_row_id", "readable_metrics", ] @@ -132,6 +133,7 @@ def _inspect_files_asserts(df: pa.Table, spark_df: DataFrame) -> None: "split_offsets", "equality_ids", "sort_order_id", + "first_row_id", ] ] rhs_subset = rhs[ @@ -145,6 +147,7 @@ def _inspect_files_asserts(df: pa.Table, spark_df: DataFrame) -> None: "split_offsets", "equality_ids", "sort_order_id", + "first_row_id", ] ] @@ -943,6 +946,7 @@ def inspect_files_asserts(df: pa.Table) -> None: "split_offsets", "equality_ids", "sort_order_id", + "first_row_id", "readable_metrics", ]