diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index d2bd4e558a..6192f13b14 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -157,6 +157,7 @@ from pyiceberg.table.deletion_vector import read_deletion_vectors from pyiceberg.table.locations import load_location_provider from pyiceberg.table.metadata import TableMetadata +from pyiceberg.table.metadata_columns import LAST_UPDATED_SEQUENCE_NUMBER, ROW_ID from pyiceberg.table.name_mapping import NameMapping, apply_name_mapping from pyiceberg.transforms import IdentityTransform, TruncateTransform from pyiceberg.typedef import EMPTY_DICT, Properties, Record, TableVersion @@ -214,6 +215,8 @@ LIST_ELEMENT_NAME = "element" MAP_KEY_NAME = "key" MAP_VALUE_NAME = "value" +# Temporary column that tracks the position of each row in its file while rows are filtered +_POSITION_COLUMN = "__iceberg_file_position" DOC = "doc" UTC_ALIASES = {"UTC", "+00:00", "Etc/UTC", "Z"} ADLS_SCHEMES = frozenset({"abfs", "abfss", "wasb", "wasbs"}) @@ -1793,12 +1796,18 @@ def _task_to_record_batches( file_project_schema = prune_columns(file_schema, projected_field_ids, select_full_types=False) + # Row IDs are derived from the position of each row in the file, so rows must not be filtered before they are counted + row_id_index = next( + (index for index, field in enumerate(projected_schema.fields) if field.field_id == ROW_ID.field_id), None + ) + filter_after_read = bool(positional_deletes) or row_id_index is not None + fragment_scanner = ds.Scanner.from_fragment( fragment=fragment, schema=physical_schema, # This will push down the query to Arrow. - # But in case there are positional deletes, we have to apply them first - filter=pyarrow_filter if not positional_deletes else None, + # But in case there are positional deletes or row IDs, rows must be counted before filtering + filter=pyarrow_filter if not filter_after_read else None, columns=[col.name for col in file_project_schema.columns], ) @@ -1808,27 +1817,36 @@ def _task_to_record_batches( next_index = next_index + len(batch) current_index = next_index - len(batch) current_batch = batch + positions = pa.array(range(current_index, next_index), type=pa.int64()) if row_id_index is not None else None if positional_deletes: # Create the mask of indices that we're interested in indices = _combine_positional_deletes(positional_deletes, current_index, current_index + len(batch)) current_batch = current_batch.take(indices) - if pyarrow_filter is not None: - # Temporary fix until PyArrow 21 is the minimum supported version - # (https://github.com/apache/arrow/pull/46057): RecordBatch.filter raises - # IndexError on PyArrow <21 when the result is empty; Table.filter does not. - table = pa.Table.from_batches([current_batch]) - table = table.filter(pyarrow_filter) - if table.num_rows == 0: - current_batch = current_batch.slice(0, 0) - else: - current_batch = table.combine_chunks().to_batches()[0] + if positions is not None: + positions = positions.take(indices) + + if filter_after_read and pyarrow_filter is not None: + # Temporary fix until PyArrow 21 is the minimum supported version + # (https://github.com/apache/arrow/pull/46057): RecordBatch.filter raises + # IndexError on PyArrow <21 when the result is empty; Table.filter does not. + table = pa.Table.from_batches([current_batch]) + if positions is not None: + table = table.append_column(_POSITION_COLUMN, positions) + table = table.filter(pyarrow_filter) + if positions is not None: + positions = table.column(_POSITION_COLUMN).combine_chunks() + table = table.drop_columns([_POSITION_COLUMN]) + if table.num_rows == 0: + current_batch = current_batch.slice(0, 0) + else: + current_batch = table.combine_chunks().to_batches()[0] # skip empty batches if current_batch.num_rows == 0: continue - yield _to_requested_schema( + result_batch = _to_requested_schema( projected_schema, file_project_schema, current_batch, @@ -1836,6 +1854,21 @@ def _task_to_record_batches( projected_missing_fields=projected_missing_fields, allow_timestamp_tz_mismatch=True, ) + if row_id_index is not None and positions is not None: + result_batch = _assign_row_ids(result_batch, row_id_index, positions, task.file.first_row_id) + yield result_batch + + +def _assign_row_ids(batch: pa.RecordBatch, row_id_index: int, positions: pa.Array, first_row_id: int | None) -> pa.RecordBatch: + """Replace null row IDs with the file's first row ID plus the row position. + + When the data file has no first row ID, every row ID is null. + """ + if first_row_id is None: + row_ids = pa.nulls(len(batch), type=pa.int64()) + else: + row_ids = pc.coalesce(batch.column(row_id_index), pc.add(positions, pa.scalar(first_row_id, type=pa.int64()))) + return batch.set_column(row_id_index, batch.schema.field(row_id_index), row_ids) @dataclass(frozen=True, slots=True) @@ -1904,6 +1937,9 @@ def __init__( *, dictionary_columns: tuple[str, ...] = (), ) -> None: + if any(field.field_id == LAST_UPDATED_SEQUENCE_NUMBER.field_id for field in projected_schema.fields): + # The data sequence number is not carried on the scan task yet + raise NotImplementedError(f"Reading {LAST_UPDATED_SEQUENCE_NUMBER.name} is not yet supported") self._table_metadata = table_metadata self._io = io self._projected_schema = projected_schema diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index cc0dbd980d..9ef7519865 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -64,6 +64,7 @@ from pyiceberg.table.locations import LocationProvider, load_location_provider from pyiceberg.table.maintenance import MaintenanceTable from pyiceberg.table.metadata import INITIAL_SEQUENCE_NUMBER, TableMetadata +from pyiceberg.table.metadata_columns import project_schema, split_selected_fields from pyiceberg.table.name_mapping import NameMapping from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef from pyiceberg.table.snapshots import ( @@ -2302,10 +2303,7 @@ def projection(self) -> Schema: else: raise ValueError(f"Snapshot not found: {self.snapshot_id}") - if "*" in self.selected_fields: - return current_schema - - return current_schema.select(*self.selected_fields, case_sensitive=self.case_sensitive) + return project_schema(current_schema, self.selected_fields, self.case_sensitive, self.table_metadata.format_version) def use_ref(self: S, name: str) -> S: if self.snapshot_id is not None: @@ -2550,9 +2548,11 @@ def _plan_files_server_side(self) -> Iterable[FileScanTask]: if self.table_identifier is None: raise ValueError("REST scan planning requires a table identifier") + # Metadata columns are not in the table schema, so the server cannot resolve them + schema_fields, _ = split_selected_fields(self.table_metadata.schema(), self.selected_fields, self.case_sensitive) request = PlanTableScanRequest( snapshot_id=self.snapshot_id, - select=list(self.selected_fields) if self.selected_fields != ("*",) else None, + select=list(schema_fields) if schema_fields != ("*",) else None, filter=self.row_filter if self.row_filter != ALWAYS_TRUE else None, case_sensitive=self.case_sensitive, ) @@ -2736,10 +2736,9 @@ def to_snapshot_id_inclusive(self: IAS, to_snapshot_id: int) -> IAS: return self.update(to_snapshot_id=to_snapshot_id) def projection(self) -> Schema: - current_schema = self.table_metadata.schema() - if "*" in self.selected_fields: - return current_schema - return current_schema.select(*self.selected_fields, case_sensitive=self.case_sensitive) + return project_schema( + self.table_metadata.schema(), self.selected_fields, self.case_sensitive, self.table_metadata.format_version + ) def plan_files(self) -> Iterable[FileScanTask]: """Plans the relevant files added between the specified snapshots.""" diff --git a/pyiceberg/table/metadata_columns.py b/pyiceberg/table/metadata_columns.py new file mode 100644 index 0000000000..c5d8bcf5da --- /dev/null +++ b/pyiceberg/table/metadata_columns.py @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Metadata columns that are not part of a table schema, but can be selected in a scan.""" + +from __future__ import annotations + +from collections.abc import Mapping +from types import MappingProxyType + +from pyiceberg.schema import Schema +from pyiceberg.types import LongType, NestedField + +ROW_ID = NestedField( + field_id=2147483540, + name="_row_id", + field_type=LongType(), + required=False, + doc="Implicit row ID that is automatically assigned", +) +LAST_UPDATED_SEQUENCE_NUMBER = NestedField( + field_id=2147483539, + name="_last_updated_sequence_number", + field_type=LongType(), + required=False, + doc="Sequence number when the row was last updated", +) + +METADATA_COLUMNS: Mapping[str, NestedField] = MappingProxyType( + {field.name: field for field in (ROW_ID, LAST_UPDATED_SEQUENCE_NUMBER)} +) + + +def _find_metadata_column(name: str, case_sensitive: bool) -> NestedField | None: + if case_sensitive: + return METADATA_COLUMNS.get(name) + return next((field for field in METADATA_COLUMNS.values() if field.name.lower() == name.lower()), None) + + +def _has_field(schema: Schema, name: str, case_sensitive: bool) -> bool: + try: + schema.find_field(name, case_sensitive=case_sensitive) + return True + except ValueError: + return False + + +def split_selected_fields( + schema: Schema, selected_fields: tuple[str, ...], case_sensitive: bool +) -> tuple[tuple[str, ...], tuple[NestedField, ...]]: + """Split selected names into schema columns and metadata columns. + + A name that resolves in the schema is a schema column, even when it matches a metadata column name. + """ + schema_names: list[str] = [] + metadata_fields: list[NestedField] = [] + for name in selected_fields: + if name != "*" and not _has_field(schema, name, case_sensitive): + if (field := _find_metadata_column(name, case_sensitive)) is not None: + if field not in metadata_fields: + metadata_fields.append(field) + continue + schema_names.append(name) + return tuple(schema_names), tuple(metadata_fields) + + +def project_schema(schema: Schema, selected_fields: tuple[str, ...], case_sensitive: bool, format_version: int) -> Schema: + """Prune the schema to the selected names, and append any selected metadata columns.""" + schema_names, metadata_fields = split_selected_fields(schema, selected_fields, case_sensitive) + + if metadata_fields and format_version < 3: + names = ", ".join(field.name for field in metadata_fields) + raise ValueError( + f"Cannot select {names}: row lineage requires format version 3, but the table has format version {format_version}" + ) + + projected = schema if "*" in schema_names else schema.select(*schema_names, case_sensitive=case_sensitive) + if not metadata_fields: + return projected + + return Schema( + *projected.fields, + *metadata_fields, + schema_id=projected.schema_id, + identifier_field_ids=projected.identifier_field_ids, + ) diff --git a/tests/integration/test_reads.py b/tests/integration/test_reads.py index e8e2dcd576..f3e55f48e6 100644 --- a/tests/integration/test_reads.py +++ b/tests/integration/test_reads.py @@ -1299,6 +1299,55 @@ def test_read_first_row_ids_written_by_spark(catalog: Catalog, spark: SparkSessi assert actual == expected +@pytest.mark.integration +@pytest.mark.parametrize("catalog", [lf("session_catalog")]) +def test_read_row_ids_written_by_spark(catalog: Catalog, spark: SparkSession) -> None: + identifier = "default.test_read_row_ids_written_by_spark" + spark.sql(f"DROP TABLE IF EXISTS {identifier}") + spark.sql( + f""" + CREATE TABLE {identifier} (id int, data string) + USING ICEBERG + TBLPROPERTIES ( + 'format-version'='3', + 'write.update.mode'='copy-on-write', + 'write.delete.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read' + ) + """ + ) + spark.sql(f"INSERT INTO {identifier} SELECT /*+ COALESCE(1) */ * FROM VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd')") + spark.sql(f"INSERT INTO {identifier} SELECT /*+ COALESCE(1) */ * FROM VALUES (5, 'e'), (6, 'f')") + # Copy-on-write rewrites the data file and stores the existing row IDs in it + spark.sql(f"UPDATE {identifier} SET data = 'x' WHERE id = 2") + # Merge-on-read adds deletion vectors, and new rows inherit their row IDs + spark.sql(f"DELETE FROM {identifier} WHERE id = 3") + spark.sql( + f""" + MERGE INTO {identifier} t + USING (SELECT * FROM VALUES (5, 'y'), (7, 'g') AS s(id, data)) s + ON t.id = s.id + WHEN MATCHED THEN UPDATE SET t.data = s.data + WHEN NOT MATCHED THEN INSERT * + """ + ) + spark.sql(f"CALL rest.system.rewrite_data_files(table => '{identifier}', options => map('rewrite-all', 'true'))") + # Rows in a new file inherit row IDs from their position, so a filter must not drop rows before they are counted + spark.sql(f"INSERT INTO {identifier} SELECT /*+ COALESCE(1) */ * FROM VALUES (8, 'h'), (9, 'i'), (10, 'j')") + + expected = sorted((row.id, row._row_id) for row in spark.sql(f"SELECT id, _row_id FROM {identifier}").collect()) + assert all(row_id is not None for _, row_id in expected) + + table = catalog.load_table(identifier) + result = table.scan(selected_fields=("id", "_row_id")).to_arrow() + assert sorted(zip(result["id"].to_pylist(), result["_row_id"].to_pylist(), strict=True)) == expected + + filtered = table.scan(row_filter="id >= 9", selected_fields=("id", "_row_id")).to_arrow() + assert sorted(zip(filtered["id"].to_pylist(), filtered["_row_id"].to_pylist(), strict=True)) == [ + (id_, row_id) for id_, row_id in expected if id_ >= 9 + ] + + @pytest.mark.integration @pytest.mark.parametrize("catalog", [lf("session_catalog_hive"), lf("session_catalog")]) def test_filter_after_arrow_scan(catalog: Catalog) -> None: diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 242cc2d495..b0f1cdde37 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -112,6 +112,7 @@ deletion_vectors_from_puffin_file, ) from pyiceberg.table.metadata import TableMetadataV2, TableMetadataV3 +from pyiceberg.table.metadata_columns import LAST_UPDATED_SEQUENCE_NUMBER, ROW_ID from pyiceberg.table.name_mapping import create_mapping_from_schema from pyiceberg.table.puffin import MAGIC_BYTES, PuffinFile from pyiceberg.transforms import BucketTransform, HourTransform, IdentityTransform, Transform, TruncateTransform @@ -1343,6 +1344,12 @@ def test_read_map(schema_map: Schema, file_map: str) -> None: ) +def test_projection_last_updated_sequence_number_not_supported(file_int: str) -> None: + schema = Schema(NestedField(1, "id", IntegerType(), required=False), LAST_UPDATED_SEQUENCE_NUMBER) + with pytest.raises(NotImplementedError, match="Reading _last_updated_sequence_number is not yet supported"): + project(schema, [file_int]) + + def test_projection_add_column_struct(schema_int: Schema, file_int: str) -> None: schema = Schema( # A new ID @@ -4436,6 +4443,64 @@ def test_task_to_record_batches_filter_after_positional_deletes_empty_result(tmp assert result_batches == [] +@pytest.mark.parametrize( + "stored_row_ids, first_row_id, deleted_positions, row_filter, expected", + [ + # Row IDs are inherited from the first row ID plus the position + (None, 100, None, AlwaysTrue(), [(1, 100), (2, 101), (3, 102), (4, 103), (5, 104)]), + # Row IDs stored in the file take precedence + ([None, 7, None, 9, None], 100, None, AlwaysTrue(), [(1, 100), (2, 7), (3, 102), (4, 9), (5, 104)]), + # Positions are counted before positional deletes are applied + (None, 100, [1, 3], AlwaysTrue(), [(1, 100), (3, 102), (5, 104)]), + # Positions are counted before the row filter is applied + (None, 100, None, GreaterThan("id", 2), [(3, 102), (4, 103), (5, 104)]), + (None, 100, [1, 3], GreaterThan("id", 2), [(3, 102), (5, 104)]), + # Without a first row ID, row IDs are null even when stored in the file + ([None, 7, None, 9, None], None, None, AlwaysTrue(), [(1, None), (2, None), (3, None), (4, None), (5, None)]), + ], +) +def test_task_to_record_batches_row_ids( + tmpdir: str, + stored_row_ids: list[int | None] | None, + first_row_id: int | None, + deleted_positions: list[int] | None, + row_filter: BooleanExpression, + expected: list[tuple[int, int | None]], +) -> None: + from pyiceberg.expressions.visitors import bind + + fields = [pa.field("id", pa.int32(), nullable=True, metadata={PYARROW_PARQUET_FIELD_ID_KEY: "1"})] + columns = [pa.array([1, 2, 3, 4, 5], type=pa.int32())] + if stored_row_ids is not None: + fields.append( + pa.field(ROW_ID.name, pa.int64(), nullable=True, metadata={PYARROW_PARQUET_FIELD_ID_KEY: str(ROW_ID.field_id)}) + ) + columns.append(pa.array(stored_row_ids, type=pa.int64())) + arrow_schema = pa.schema(fields) + data_file = _write_table_to_data_file(f"{tmpdir}/row_ids.parquet", arrow_schema, pa.table(columns, schema=arrow_schema)) + data_file.first_row_id = first_row_id + + table_schema = Schema(NestedField(1, "id", IntegerType(), required=False)) + projected_schema = Schema(*table_schema.fields, ROW_ID) + positional_deletes = [pa.chunked_array([pa.array(deleted_positions, type=pa.int64())])] if deleted_positions else None + result_batches = list( + _task_to_record_batches( + PyArrowFileIO(), + FileScanTask(data_file), + bound_row_filter=bind(table_schema, row_filter, case_sensitive=True), + projected_schema=projected_schema, + table_schema=table_schema, + projected_field_ids={1, ROW_ID.field_id}, + positional_deletes=positional_deletes, + case_sensitive=True, + ) + ) + + result = pa.Table.from_batches(result_batches) + assert result.schema.field(ROW_ID.name).type == pa.int64() + assert list(zip(result["id"].to_pylist(), result[ROW_ID.name].to_pylist(), strict=True)) == expected + + def test_task_to_record_batches_encrypted_by_iceberg_rust() -> None: """The data file was encrypted by iceberg-rust, see tests/table/encryption/README.md.""" data_file = DataFile.from_args( diff --git a/tests/table/test_init.py b/tests/table/test_init.py index 5a90a80d83..af9a11a452 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -50,6 +50,7 @@ ) from pyiceberg.table.labels import Labels from pyiceberg.table.metadata import EncryptedKey, TableMetadataUtil, TableMetadataV1, TableMetadataV2, _generate_snapshot_id +from pyiceberg.table.metadata_columns import LAST_UPDATED_SEQUENCE_NUMBER, ROW_ID, project_schema, split_selected_fields from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef, SnapshotRefType from pyiceberg.table.snapshots import ( MetadataLogEntry, @@ -390,6 +391,51 @@ def test_table_scan_projection_unknown_column(table_v2: Table) -> None: assert "Could not find column: 'a'" in str(exc_info.value) +def test_table_scan_projection_row_lineage_columns(table_v3: Table) -> None: + projection_schema = table_v3.scan().select("x", "_row_id", "_last_updated_sequence_number").projection() + assert projection_schema == Schema( + NestedField(field_id=1, name="x", field_type=LongType(), required=True), + ROW_ID, + LAST_UPDATED_SEQUENCE_NUMBER, + identifier_field_ids=[1], + ) + assert projection_schema.schema_id == 1 + + +def test_table_scan_projection_row_lineage_columns_with_star(table_v3: Table) -> None: + projection_schema = table_v3.scan(selected_fields=("*", "_row_id")).projection() + assert projection_schema.fields == (*table_v3.schema().fields, ROW_ID) + + +def test_table_scan_projection_row_lineage_columns_case_insensitive(table_v3: Table) -> None: + projection_schema = table_v3.scan().with_case_sensitive(False).select("X", "_ROW_ID").projection() + assert [field.name for field in projection_schema.fields] == ["x", "_row_id"] + + +def test_table_scan_projection_row_lineage_columns_case_sensitive(table_v3: Table) -> None: + with pytest.raises(ValueError, match="Could not find column: '_ROW_ID'"): + table_v3.scan().select("x", "_ROW_ID").projection() + + +def test_table_scan_projection_row_lineage_columns_before_v3(table_v2: Table) -> None: + with pytest.raises(ValueError, match="Cannot select _row_id: row lineage requires format version 3"): + table_v2.scan().select("x", "_row_id").projection() + + +def test_incremental_append_scan_projection_row_lineage_columns(table_v3: Table) -> None: + projection_schema = table_v3.incremental_append_scan(selected_fields=("x", "_row_id")).projection() + assert [field.name for field in projection_schema.fields] == ["x", "_row_id"] + + +def test_project_schema_prefers_schema_column_over_metadata_column() -> None: + schema = Schema(NestedField(field_id=1, name="_row_id", field_type=LongType(), required=True)) + assert project_schema(schema, ("_row_id",), case_sensitive=True, format_version=3) == schema + assert split_selected_fields(schema, ("_row_id", "_last_updated_sequence_number"), case_sensitive=True) == ( + ("_row_id",), + (LAST_UPDATED_SEQUENCE_NUMBER,), + ) + + def test_data_scan_plan_files_no_current_snapshot(example_table_metadata_no_snapshot_v1: dict[str, Any]) -> None: table = Table( identifier=("default", "test_no_snapshot"),