From 3c1a26ccae96aa6fbff622d662ca563a03ac4584 Mon Sep 17 00:00:00 2001 From: Ayush Kumar Saha <225407166+AP24110011695@users.noreply.github.com> Date: Wed, 18 Nov 2026 00:12:19 +0530 Subject: [PATCH] Populate nan_value_counts when writing parquet files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit data_file_statistics_from_parquet_metadata returned an empty nan_value_counts dict unconditionally, so data files written by the pyarrow writer never reported NaN counts and NaN-aware scan pruning stayed unavailable for them — the metrics evaluator degrades to ROWS_MIGHT_MATCH without nan counts (_InclusiveMetricsEvaluator visit_is_nan / _contains_nans_only). pyarrow's statistics API does not expose NaN counts, so the writer now counts them from the written table itself (pa.compute.is_nan summed per top-level float/double field, excluding nulls per the spec's NaN-count semantics) and passes them through a new optional table_nan_value_counts parameter. The metadata-only caller (parquet_file_to_data_file) keeps its previous behavior. Refs #4396. Signed-off-by: Ayush Kumar Saha <225407166+AP24110011695@users.noreply.github.com> --- pyiceberg/io/pyarrow.py | 16 ++++++++- tests/io/test_pyarrow_stats.py | 62 +++++++++++++++++++++++++++++++++- 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 53db9cd6b7..0922fdd6fa 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -2721,6 +2721,7 @@ def data_file_statistics_from_parquet_metadata( parquet_metadata: pq.FileMetaData, stats_columns: dict[int, StatisticsCollector], parquet_column_mapping: dict[str, int], + table_nan_value_counts: dict[int, int] | None = None, ) -> DataFileStatistics: """ Compute and return DataFileStatistics that includes the following. @@ -2738,6 +2739,9 @@ def data_file_statistics_from_parquet_metadata( stats_columns (Dict[int, StatisticsCollector]): The statistics gathering plan. It is required to set the mode for column metrics collection parquet_column_mapping (Dict[str, int]): The mapping of the parquet file name to the field ID + table_nan_value_counts (Dict[int, int] | None): NaN counts computed from the written table itself. + The parquet statistics API does not expose NaN counts, so the writer supplies them; defaults + to no NaN counts being reported. """ column_sizes: dict[int, int] = {} value_counts: dict[int, int] = {} @@ -2845,7 +2849,7 @@ def data_file_statistics_from_parquet_metadata( column_sizes=column_sizes, value_counts=value_counts, null_value_counts=null_value_counts, - nan_value_counts=nan_value_counts, + nan_value_counts=table_nan_value_counts if table_nan_value_counts is not None else nan_value_counts, column_aggregates=col_aggs, split_offsets=split_offsets, ) @@ -2866,6 +2870,7 @@ def __init__( self._properties = properties self._writer: pq.ParquetWriter | None = None self._fos: OutputStream | None = None + self._nan_value_counts: dict[int, int] = {} self._parquet_writer_kwargs = _get_parquet_writer_kwargs(properties) self._parquet_writer_kwargs["encryption_properties"] = _parquet_encryption_properties(key_metadata) self._row_group_size = property_as_int( @@ -2889,6 +2894,14 @@ def write(self, table: pa.Table) -> None: raise self._fos = fos self._writer.write(table, row_group_size=self._row_group_size) + # The parquet statistics API does not expose NaN counts, so they are counted from the data + # itself. `is_nan` is null on nulls, and `sum` skips nulls, matching the spec's NaN-count + # semantics of counting non-null NaN values. + for field in self._file_schema.fields: + if isinstance(field.field_type, (FloatType, DoubleType)): + nan_count = pc.sum(pc.is_nan(table.column(field.name)), min_count=0).as_py() + if nan_count: + self._nan_value_counts[field.field_id] = self._nan_value_counts.get(field.field_id, 0) + nan_count def close(self) -> DataFileStatistics: if self._result is not None: @@ -2901,6 +2914,7 @@ def close(self) -> DataFileStatistics: parquet_metadata=self._writer.writer.metadata, stats_columns=compute_statistics_plan(self._file_schema, self._properties), parquet_column_mapping=parquet_path_to_id_mapping(self._file_schema), + table_nan_value_counts=self._nan_value_counts, ) return self._result diff --git a/tests/io/test_pyarrow_stats.py b/tests/io/test_pyarrow_stats.py index 1c5e802a26..8c95dbf713 100644 --- a/tests/io/test_pyarrow_stats.py +++ b/tests/io/test_pyarrow_stats.py @@ -44,11 +44,13 @@ STRUCT_INT32, STRUCT_INT64, ) -from pyiceberg.expressions import IsNull +from pyiceberg.expressions import IsNull, NotNaN from pyiceberg.expressions.visitors import _InclusiveMetricsEvaluator from pyiceberg.io.pyarrow import ( MetricModeTypes, MetricsMode, + ParquetFormatWriter, + PyArrowFileIO, PyArrowStatisticsCollector, compute_statistics_plan, data_file_statistics_from_parquet_metadata, @@ -67,6 +69,7 @@ from pyiceberg.typedef import Record from pyiceberg.types import ( BooleanType, + DoubleType, FloatType, IntegerType, NestedField, @@ -325,6 +328,63 @@ def test_unknown_row_group_null_count_is_omitted(unknown_row_group: int) -> None assert _InclusiveMetricsEvaluator(schema, IsNull("value")).eval(data_file) +def test_nan_value_counts_are_written(tmp_path) -> None: + schema = Schema(NestedField(1, "reading", DoubleType(), required=False)) + arrow_schema = schema_to_pyarrow(schema) + table = pa.table( + {"reading": pa.array([1.5, float("nan"), float("nan"), None], type=pa.float64())}, + schema=arrow_schema, + ) + + writer = ParquetFormatWriter(PyArrowFileIO().new_output(str(tmp_path / "nan.parquet")), file_schema=schema, properties={}) + writer.write(table) + statistics = writer.close() + + # Two non-null NaNs; nulls are counted separately and excluded from the NaN count + assert statistics.nan_value_counts == {1: 2} + + data_file = DataFile.from_args( + file_path="nan.parquet", + file_format=FileFormat.PARQUET, + partition=Record(), + file_size_in_bytes=1, + **statistics.to_serialized_dict(), + ) + # The file holds non-NaN values, so a not_nan filter must not prune it + assert _InclusiveMetricsEvaluator(schema, NotNaN("reading")).eval(data_file) + + +def test_nan_value_counts_are_written_across_row_groups(tmp_path) -> None: + schema = Schema(NestedField(1, "reading", DoubleType(), required=False)) + arrow_schema = schema_to_pyarrow(schema) + table = pa.table( + { + "reading": pa.array( + [float("nan"), float("nan")], + type=pa.float64(), + ) + }, + schema=arrow_schema, + ) + + writer = ParquetFormatWriter(PyArrowFileIO().new_output(str(tmp_path / "allnan.parquet")), file_schema=schema, properties={}) + writer.write(table.slice(0, 1)) + writer.write(table.slice(1, 1)) + statistics = writer.close() + + assert statistics.nan_value_counts == {1: 2} + + data_file = DataFile.from_args( + file_path="allnan.parquet", + file_format=FileFormat.PARQUET, + partition=Record(), + file_size_in_bytes=1, + **statistics.to_serialized_dict(), + ) + # Every non-null value is NaN, so a not_nan filter prunes the file + assert not _InclusiveMetricsEvaluator(schema, NotNaN("reading")).eval(data_file) + + def test_bounds() -> None: metadata, table_metadata = construct_test_table()