diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 53db9cd6b7..42fc789679 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -2717,10 +2717,45 @@ def parquet_path_to_id_mapping( return result +def _nan_value_counts_from_arrow(table: pa.Table, stats_columns: dict[int, StatisticsCollector]) -> dict[int, int]: + """Count the non-null NaN values of the top-level float and double columns. + + PyArrow does not expose the NaN count of a Parquet column statistic, so the counts are taken + from the data before it is written. Columns that cannot be counted are left out, which only + means the file cannot be pruned on NaN counts. + + Args: + table (pyarrow.Table): The data that is written to the file. + stats_columns (dict[int, StatisticsCollector]): The statistics gathering plan. + + Returns: + A mapping of Iceberg field ID to the number of non-null NaN values. + """ + nan_value_counts: dict[int, int] = {} + for pos, arrow_field in enumerate(table.schema): + if (field_id := _get_field_id(arrow_field)) is None: + continue + + stats_col = stats_columns.get(field_id) + if stats_col is None: + continue + + if stats_col.mode == MetricsMode(MetricModeTypes.NONE): + continue + + if not isinstance(stats_col.iceberg_type, (FloatType, DoubleType)): + continue + + nan_value_counts[field_id] = pc.sum(pc.is_nan(table.column(pos)), min_count=0).as_py() + + return nan_value_counts + + def data_file_statistics_from_parquet_metadata( parquet_metadata: pq.FileMetaData, stats_columns: dict[int, StatisticsCollector], parquet_column_mapping: dict[str, int], + nan_value_counts: dict[int, int] | None = None, ) -> DataFileStatistics: """ Compute and return DataFileStatistics that includes the following. @@ -2738,13 +2773,14 @@ def data_file_statistics_from_parquet_metadata( stats_columns (Dict[int, StatisticsCollector]): The statistics gathering plan. It is required to set the mode for column metrics collection parquet_column_mapping (Dict[str, int]): The mapping of the parquet file name to the field ID + nan_value_counts (dict[int, int] | None): The NaN value counts computed from the data, if available """ column_sizes: dict[int, int] = {} value_counts: dict[int, int] = {} split_offsets: list[int] = [] null_value_counts: dict[int, int] = {} - nan_value_counts: dict[int, int] = {} + nan_value_counts = dict(nan_value_counts or {}) col_aggs = {} @@ -2863,11 +2899,12 @@ def __init__( ) -> None: self._output_file = output_file self._file_schema = file_schema - self._properties = properties self._writer: pq.ParquetWriter | None = None self._fos: OutputStream | None = None self._parquet_writer_kwargs = _get_parquet_writer_kwargs(properties) self._parquet_writer_kwargs["encryption_properties"] = _parquet_encryption_properties(key_metadata) + self._stats_columns = compute_statistics_plan(file_schema, properties) + self._nan_value_counts: dict[int, int] = {} self._row_group_size = property_as_int( properties=properties, property_name=TableProperties.PARQUET_ROW_GROUP_LIMIT, @@ -2889,6 +2926,8 @@ def write(self, table: pa.Table) -> None: raise self._fos = fos self._writer.write(table, row_group_size=self._row_group_size) + for field_id, count in _nan_value_counts_from_arrow(table, self._stats_columns).items(): + self._nan_value_counts[field_id] = self._nan_value_counts.get(field_id, 0) + count def close(self) -> DataFileStatistics: if self._result is not None: @@ -2899,8 +2938,9 @@ def close(self) -> DataFileStatistics: self._writer.close() self._result = data_file_statistics_from_parquet_metadata( parquet_metadata=self._writer.writer.metadata, - stats_columns=compute_statistics_plan(self._file_schema, self._properties), + stats_columns=self._stats_columns, parquet_column_mapping=parquet_path_to_id_mapping(self._file_schema), + nan_value_counts=self._nan_value_counts, ) return self._result diff --git a/tests/integration/test_writes/test_writes.py b/tests/integration/test_writes/test_writes.py index fe720440fb..e0d647c2e9 100644 --- a/tests/integration/test_writes/test_writes.py +++ b/tests/integration/test_writes/test_writes.py @@ -1688,7 +1688,7 @@ def test_merge_manifests_file_content(session_catalog: Catalog, arrow_table_with (11, b"\x01"), (12, b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"), ] - assert tbl_a_data_file["nan_value_counts"] == [] + assert tbl_a_data_file["nan_value_counts"] == [(6, 0), (7, 0)] assert tbl_a_data_file["null_value_counts"] == [ (1, 1), (2, 1), diff --git a/tests/io/test_format_writers.py b/tests/io/test_format_writers.py index 86ae65156b..cf6764b1c2 100644 --- a/tests/io/test_format_writers.py +++ b/tests/io/test_format_writers.py @@ -24,10 +24,10 @@ import pytest from pyiceberg.io.fileformat import FileFormatFactory, FileFormatModel -from pyiceberg.io.pyarrow import PyArrowFileIO +from pyiceberg.io.pyarrow import ParquetFormatModel, PyArrowFileIO, schema_to_pyarrow from pyiceberg.manifest import FileFormat from pyiceberg.schema import Schema -from pyiceberg.types import LongType, NestedField +from pyiceberg.types import DoubleType, FloatType, LongType, NestedField @pytest.fixture(params=FileFormatFactory.available_formats(), ids=lambda f: f.name.lower()) @@ -138,3 +138,58 @@ def test_parquet_format_model_adds_field_id_metadata() -> None: metadata_no_ids: dict[bytes, bytes] = {} ParquetFormatModel().add_field_metadata(field, metadata_no_ids, include_field_ids=False) assert metadata_no_ids == {} + + +def test_parquet_writer_accumulates_nan_value_counts(tmp_path: Path) -> None: + """NaN counts from multiple write() calls are added together.""" + schema = Schema(NestedField(field_id=1, name="d", field_type=DoubleType(), required=False)) + arrow_schema = schema_to_pyarrow(schema, include_field_ids=True) + file_path = str(tmp_path / "test.parquet") + writer = ParquetFormatModel().create_writer(PyArrowFileIO().new_output(file_path), schema, {}) + + writer.write(pa.Table.from_arrays([pa.array([float("nan"), float("nan")], type=pa.float64())], schema=arrow_schema)) + writer.write(pa.Table.from_arrays([pa.array([1.0, None], type=pa.float64())], schema=arrow_schema)) + statistics = writer.close() + + assert dict(statistics.nan_value_counts) == {1: 2} + + +def test_parquet_writer_skips_nan_counts_for_none_metrics_mode(tmp_path: Path) -> None: + """A column with metrics mode none does not report a NaN count.""" + schema = Schema( + NestedField(field_id=1, name="f", field_type=FloatType(), required=False), + NestedField(field_id=2, name="d", field_type=DoubleType(), required=False), + ) + arrow_schema = schema_to_pyarrow(schema, include_field_ids=True) + file_path = str(tmp_path / "test.parquet") + writer = ParquetFormatModel().create_writer( + PyArrowFileIO().new_output(file_path), schema, {"write.metadata.metrics.column.f": "none"} + ) + + writer.write( + pa.Table.from_arrays( + [ + pa.array([float("nan"), 1.0], type=pa.float32()), + pa.array([float("nan"), 1.0], type=pa.float64()), + ], + schema=arrow_schema, + ) + ) + statistics = writer.close() + + assert dict(statistics.nan_value_counts) == {2: 1} + + +def test_parquet_writer_default_none_metrics_mode_reports_no_nan_counts(tmp_path: Path) -> None: + """With metrics mode none as the default, no column reports a NaN count.""" + schema = Schema(NestedField(field_id=1, name="d", field_type=DoubleType(), required=False)) + arrow_schema = schema_to_pyarrow(schema, include_field_ids=True) + file_path = str(tmp_path / "test.parquet") + writer = ParquetFormatModel().create_writer( + PyArrowFileIO().new_output(file_path), schema, {"write.metadata.metrics.default": "none"} + ) + + writer.write(pa.Table.from_arrays([pa.array([float("nan"), 1.0], type=pa.float64())], schema=arrow_schema)) + statistics = writer.close() + + assert dict(statistics.nan_value_counts) == {} diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index b0f1cdde37..9341cfdd46 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -3596,6 +3596,39 @@ def test_append_to_table_bucketed_by_time(catalog: InMemoryCatalog) -> None: assert table.scan(row_filter=EqualTo("event_time", "22:31:08")).to_arrow()["id"].to_pylist() == [2] +def test_append_counts_nan_values(catalog: InMemoryCatalog) -> None: + schema = Schema( + NestedField(id=1, name="f", field_type=FloatType(), required=False), + NestedField(id=2, name="d", field_type=DoubleType(), required=False), + ) + catalog.create_namespace("default") + table = catalog.create_table("default.test_append_nan_counts", schema=schema) + + table.append( + pa.table( + { + "f": pa.array([float("nan"), float("nan"), 1.0, None], type=pa.float32()), + "d": pa.array([float("nan"), 2.0, None, float("nan")], type=pa.float64()), + } + ) + ) + + data_file = next(iter(table.scan().plan_files())).file + assert dict(data_file.nan_value_counts) == {1: 2, 2: 2} + + +def test_scan_prunes_files_without_nan_values(catalog: InMemoryCatalog) -> None: + schema = Schema(NestedField(id=1, name="d", field_type=DoubleType(), required=False)) + catalog.create_namespace("default") + table = catalog.create_table("default.test_scan_prunes_nan_counts", schema=schema) + + table.append(pa.table({"d": pa.array([float("nan"), 1.0], type=pa.float64())})) + table.append(pa.table({"d": pa.array([2.0, 3.0], type=pa.float64())})) + + assert len(list(table.scan(row_filter="d IS NAN").plan_files())) == 1 + assert len(list(table.scan(row_filter="d IS NOT NAN").plan_files())) == 2 + + def test_inspect_partition_for_nested_field(catalog: InMemoryCatalog) -> None: schema = Schema( NestedField(id=1, name="foo", field_type=StringType(), required=True),