From 929b1e597326d94be47c9e9cd8bc0e201c713a1d Mon Sep 17 00:00:00 2001 From: Gedalia Kliger <117381134+gedaliakliger@users.noreply.github.com> Date: Tue, 17 Nov 2026 05:11:48 +0000 Subject: [PATCH 1/2] Read a list or map column referenced by a null filter A scan filter that tests a list or map column for null added the column's own field ID to the projected IDs, and prune_columns rejects a list or map selected by its own ID. Project the column's nested fields instead. --- pyiceberg/io/pyarrow.py | 8 +++++++- tests/io/test_pyarrow.py | 41 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 53db9cd6b7..5961da883c 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -1952,11 +1952,17 @@ def __init__( @property def _projected_field_ids(self) -> set[int]: """Set of field IDs that should be projected from the data files.""" + schema = self._table_metadata.schema() + filter_field_ids = extract_field_ids(self._bound_row_filter) + # A list or map cannot be projected by its own ID, so a filter on one reads its nested fields instead + nested_field_ids = prune_columns( + schema, {id for id in filter_field_ids if isinstance(schema.find_type(id), (MapType, ListType))} + ).field_ids return { id for id in self._projected_schema.field_ids if not isinstance(self._projected_schema.find_type(id), (MapType, ListType)) - }.union(extract_field_ids(self._bound_row_filter)) + }.union(id for id in filter_field_ids | nested_field_ids if not isinstance(schema.find_type(id), (MapType, ListType))) def to_table(self, tasks: Iterable[FileScanTask]) -> pa.Table: """Scan the Iceberg table and return a pa.Table. diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index b0f1cdde37..f00646ed6c 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -72,6 +72,7 @@ GreaterThan, IsNull, Not, + NotNull, Or, ) from pyiceberg.expressions.literals import literal @@ -1953,6 +1954,46 @@ def test_projection_filter_on_unprojected_field(schema_int_str: Schema, file_int assert repr(result_table.schema) == "id: int32 not null" +def test_projection_null_filter_on_list_and_map(tmpdir: str) -> None: + schema = Schema( + NestedField(1, "id", IntegerType(), required=False), + NestedField(2, "tags", ListType(21, StringType(), element_required=False), required=False), + NestedField( + 3, + "props", + MapType(key_id=31, key_type=StringType(), value_id=32, value_type=IntegerType(), value_required=False), + required=False, + ), + NestedField( + 4, + "points", + ListType(41, StructType(NestedField(42, "x", IntegerType(), required=False)), element_required=False), + required=False, + ), + ) + pyarrow_schema = schema_to_pyarrow(schema, metadata={ICEBERG_SCHEMA: bytes(schema.model_dump_json(), UTF8)}) + file = _write_table_to_file( + f"{tmpdir}/null_list_map.parquet", + pyarrow_schema, + pa.Table.from_pylist( + [ + {"id": 1, "tags": ["a"], "props": [("k", 1)], "points": [{"x": 1}]}, + {"id": 2, "tags": [], "props": [], "points": []}, + {"id": 3, "tags": None, "props": None, "points": None}, + ], + schema=pyarrow_schema, + ), + ) + schema_id = Schema(NestedField(1, "id", IntegerType(), required=False)) + + for column in ("tags", "props", "points"): + for expr, expected in ((IsNull(column), [3]), (NotNull(column), [1, 2])): + assert project(schema, [file], expr)["id"].to_pylist() == expected + assert project(schema_id, [file], expr, schema).to_pydict() == {"id": expected} + + assert project(schema, [file], IsNull("tags")).to_pylist() == [{"id": 3, "tags": None, "props": None, "points": None}] + + def test_projection_filter_on_unknown_field(schema_int_str: Schema, file_int_str: str) -> None: schema = Schema(NestedField(1, "id", IntegerType())) From 9a74899d92550a1b447b645d138accd8dad8ba29 Mon Sep 17 00:00:00 2001 From: Gedalia Kliger <117381134+gedaliakliger@users.noreply.github.com> Date: Tue, 17 Nov 2026 16:26:55 +0000 Subject: [PATCH 2/2] Test null filters on list and map columns through a scan --- tests/catalog/test_catalog_behaviors.py | 19 ++++++++++++ tests/io/test_pyarrow.py | 41 ------------------------- 2 files changed, 19 insertions(+), 41 deletions(-) diff --git a/tests/catalog/test_catalog_behaviors.py b/tests/catalog/test_catalog_behaviors.py index cd817512b4..4b8fea113e 100644 --- a/tests/catalog/test_catalog_behaviors.py +++ b/tests/catalog/test_catalog_behaviors.py @@ -1429,6 +1429,25 @@ def test_scan_integer_column_with_decimal_literal(catalog: Catalog) -> None: tbl.scan(row_filter="x > 2.6").to_arrow() +def test_scan_null_filter_on_list_and_map_columns(catalog: Catalog) -> None: + catalog.create_namespace("default") + data = pa.table( + { + "id": pa.array([1, 2, 3], pa.int32()), + "tags": pa.array([["a"], [], None], pa.list_(pa.string())), + "props": pa.array([[("k", 1)], [], None], pa.map_(pa.string(), pa.int32())), + "points": pa.array([[{"x": 1}], [], None], pa.list_(pa.struct([("x", pa.int32())]))), + } + ) + tbl = catalog.create_table(f"default.scan_null_filter_list_map_{catalog.name}", schema=data.schema) + tbl.append(data) + + for column in ("tags", "props", "points"): + assert tbl.scan(row_filter=f"{column} is null").to_arrow()["id"].to_pylist() == [3], column + assert sorted(tbl.scan(row_filter=f"{column} is not null").to_arrow()["id"].to_pylist()) == [1, 2], column + assert tbl.scan(row_filter=f"{column} is null", selected_fields=("id",)).to_arrow().to_pydict() == {"id": [3]}, column + + def test_scan_negated_comparison_matches_file_pruning_for_nan(catalog: Catalog) -> None: catalog.create_namespace("default") identifier = f"default.scan_negated_comparison_nan_{catalog.name}" diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index f00646ed6c..b0f1cdde37 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -72,7 +72,6 @@ GreaterThan, IsNull, Not, - NotNull, Or, ) from pyiceberg.expressions.literals import literal @@ -1954,46 +1953,6 @@ def test_projection_filter_on_unprojected_field(schema_int_str: Schema, file_int assert repr(result_table.schema) == "id: int32 not null" -def test_projection_null_filter_on_list_and_map(tmpdir: str) -> None: - schema = Schema( - NestedField(1, "id", IntegerType(), required=False), - NestedField(2, "tags", ListType(21, StringType(), element_required=False), required=False), - NestedField( - 3, - "props", - MapType(key_id=31, key_type=StringType(), value_id=32, value_type=IntegerType(), value_required=False), - required=False, - ), - NestedField( - 4, - "points", - ListType(41, StructType(NestedField(42, "x", IntegerType(), required=False)), element_required=False), - required=False, - ), - ) - pyarrow_schema = schema_to_pyarrow(schema, metadata={ICEBERG_SCHEMA: bytes(schema.model_dump_json(), UTF8)}) - file = _write_table_to_file( - f"{tmpdir}/null_list_map.parquet", - pyarrow_schema, - pa.Table.from_pylist( - [ - {"id": 1, "tags": ["a"], "props": [("k", 1)], "points": [{"x": 1}]}, - {"id": 2, "tags": [], "props": [], "points": []}, - {"id": 3, "tags": None, "props": None, "points": None}, - ], - schema=pyarrow_schema, - ), - ) - schema_id = Schema(NestedField(1, "id", IntegerType(), required=False)) - - for column in ("tags", "props", "points"): - for expr, expected in ((IsNull(column), [3]), (NotNull(column), [1, 2])): - assert project(schema, [file], expr)["id"].to_pylist() == expected - assert project(schema_id, [file], expr, schema).to_pydict() == {"id": expected} - - assert project(schema, [file], IsNull("tags")).to_pylist() == [{"id": 3, "tags": None, "props": None, "points": None}] - - def test_projection_filter_on_unknown_field(schema_int_str: Schema, file_int_str: str) -> None: schema = Schema(NestedField(1, "id", IntegerType()))