From aa2bc68da106a929ae835fbd3e77521645978860 Mon Sep 17 00:00:00 2001 From: comphead Date: Tue, 27 Oct 2026 09:16:14 -0700 Subject: [PATCH 1/7] fix(reader): keep matching rows when filtering a promoted column Predicates are bound to the table schema, but the row filter evaluated them on batches in the file's physical types. For a file written before an int -> long, float -> double, or decimal precision promotion, try_cast_literal cast the literal down to the file type, so an out-of-range literal became null and dropped every row, and a double literal was rounded to f32. Convert the filter columns to the table schema with a RecordBatchTransformer, as the reader does for its output, and make try_cast_literal fail instead of returning null. Page-index pruning is left to #3507 and #3521. Part of #3122. --- crates/iceberg/src/arrow/reader/pipeline.rs | 1 + .../src/arrow/reader/predicate_visitor.rs | 61 +++++++-- crates/iceberg/src/arrow/reader/row_filter.rs | 123 ++++++++++++++++-- 3 files changed, 164 insertions(+), 21 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs index 132ad98ce7..540982617b 100644 --- a/crates/iceberg/src/arrow/reader/pipeline.rs +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -580,6 +580,7 @@ impl FileScanTaskReader { record_batch_stream_builder.parquet_schema(), &iceberg_field_ids, &field_id_map, + task.schema_ref(), )?; record_batch_stream_builder = record_batch_stream_builder.with_row_filter(row_filter); diff --git a/crates/iceberg/src/arrow/reader/predicate_visitor.rs b/crates/iceberg/src/arrow/reader/predicate_visitor.rs index fca0e90541..45a3d49983 100644 --- a/crates/iceberg/src/arrow/reader/predicate_visitor.rs +++ b/crates/iceberg/src/arrow/reader/predicate_visitor.rs @@ -27,7 +27,7 @@ use arrow_array::cast::AsArray; use arrow_array::types::{Float32Type, Float64Type}; use arrow_array::{Array, ArrayRef, BooleanArray, Datum as ArrowDatum, RecordBatch, Scalar}; use arrow_buffer::BooleanBuffer; -use arrow_cast::cast::cast; +use arrow_cast::cast::{CastOptions, cast_with_options}; use arrow_ord::cmp::{eq, gt, gt_eq, lt, lt_eq, neq}; use arrow_schema::{ArrowError, DataType}; use arrow_string::like::starts_with; @@ -885,12 +885,10 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { } } -/// The Arrow type of an array that the Parquet reader reads may not match the exact Arrow type -/// that Iceberg uses for literals - but they are effectively the same logical type, -/// i.e. LargeUtf8 and Utf8 or Utf8View and Utf8 or Utf8View and LargeUtf8. -/// -/// The Arrow compute kernels that we use must match the type exactly, so first cast the literal -/// into the type of the batch we read from Parquet before sending it to the compute kernel. +/// The row filter evaluates batches converted to the table schema, but a literal can still use +/// another Arrow representation of the same type (e.g. `Binary` for a `LargeBinary` column). +/// The compute kernels need the exact type, so cast the literal, failing rather than letting a +/// lossy cast turn it into null. fn try_cast_literal( literal: &Arc, column_type: &DataType, @@ -902,7 +900,11 @@ fn try_cast_literal( return Ok(Arc::clone(literal)); } - let literal_array = cast(literal_array, column_type)?; + let options = CastOptions { + safe: false, + ..Default::default() + }; + let literal_array = cast_with_options(literal_array, column_type, &options)?; Ok(Arc::new(Scalar::new(literal_array))) } @@ -911,8 +913,8 @@ mod tests { use std::collections::{HashMap, HashSet}; use std::sync::Arc; - use arrow_array::{Array, BooleanArray, RecordBatch}; - use arrow_schema::{DataType, Field, Schema as ArrowSchema}; + use arrow_array::{Array, BooleanArray, Int32Array, RecordBatch}; + use arrow_schema::{ArrowError, DataType, Field, Schema as ArrowSchema}; use parquet::schema::parser::parse_message_type; use parquet::schema::types::SchemaDescriptor; @@ -1337,4 +1339,43 @@ mod tests { assert_eq!(residual, expected, "{name}"); } } + + /// A literal that does not fit the column type fails the predicate instead of becoming null. + #[test] + fn test_predicate_converter_rejects_lossy_literal_cast() { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(), + ); + let predicate = Reference::new("x") + .less_than(Datum::long(i64::from(i32::MAX) + 1)) + .bind(schema, true) + .unwrap(); + let parquet_type = parse_message_type("message schema { required int32 x = 1; }").unwrap(); + let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_type)); + let column_map = HashMap::from([(1, 0)]); + let column_indices = vec![0]; + let mut converter = PredicateConverter { + parquet_schema: &parquet_schema, + column_map: &column_map, + column_indices: &column_indices, + }; + let mut predicate_fn = visit(&mut converter, &predicate).unwrap(); + + // An `int` batch that was not converted to the table's `long`. + let batch = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![Field::new( + "x", + DataType::Int32, + false, + )])), + vec![Arc::new(Int32Array::from(vec![1]))], + ) + .unwrap(); + assert!(matches!(predicate_fn(batch), Err(ArrowError::CastError(_)))); + } } diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs index 1bbdf9f60c..253e39b12d 100644 --- a/crates/iceberg/src/arrow/reader/row_filter.rs +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -23,18 +23,20 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; +use arrow_schema::ArrowError; use parquet::arrow::ProjectionMask; use parquet::arrow::arrow_reader::{ArrowPredicateFn, RowFilter, RowSelection}; use parquet::file::metadata::ParquetMetaData; use parquet::schema::types::SchemaDescriptor; use super::{ArrowReader, PredicateConverter}; +use crate::arrow::record_batch_transformer::RecordBatchTransformerBuilder; use crate::error::Result; use crate::expr::BoundPredicate; use crate::expr::visitors::bound_predicate_visitor::visit; use crate::expr::visitors::page_index_evaluator::PageIndexEvaluator; use crate::expr::visitors::row_group_metrics_evaluator::RowGroupMetricsEvaluator; -use crate::spec::Schema; +use crate::spec::{Schema, SchemaRef}; use crate::{Error, ErrorKind}; impl ArrowReader { @@ -43,14 +45,20 @@ impl ArrowReader { parquet_schema: &SchemaDescriptor, iceberg_field_ids: &HashSet, field_id_map: &HashMap, + snapshot_schema: SchemaRef, ) -> Result { - // Collect Parquet column indices from field ids. + // Collect Parquet column indices from field ids, keeping each column's field id. // If the field id is not found in Parquet schema, it will be ignored due to schema evolution. - let mut column_indices = iceberg_field_ids - .iter() - .filter_map(|field_id| field_id_map.get(field_id).cloned()) - .collect::>(); - column_indices.sort(); + let mut columns = Vec::with_capacity(iceberg_field_ids.len()); + columns.extend( + iceberg_field_ids + .iter() + .filter_map(|field_id| field_id_map.get(field_id).map(|&idx| (idx, *field_id))), + ); + // Leaf order is the filter batch's column order, so the transformer below passes the + // batch through unchanged when no column needs converting. + columns.sort_unstable(); + let (column_indices, field_ids): (Vec<_>, Vec<_>) = columns.into_iter().unzip(); // The converter that converts `BoundPredicates` to `ArrowPredicates` let mut converter = PredicateConverter { @@ -62,8 +70,18 @@ impl ArrowReader { // After collecting required leaf column indices used in the predicate, // creates the projection mask for the Arrow predicates. let projection_mask = ProjectionMask::leaves(parquet_schema, column_indices.clone()); - let predicate_func = visit(&mut converter, predicates)?; - let arrow_predicate = ArrowPredicateFn::new(projection_mask, predicate_func); + let mut predicate_func = visit(&mut converter, predicates)?; + // Predicates are bound to the table schema, so evaluate them on the filter columns + // converted to it, as the reader's output is (e.g. a file written before an `int` -> + // `long` promotion). + let mut transformer = + RecordBatchTransformerBuilder::new(snapshot_schema, &field_ids).build(); + let arrow_predicate = ArrowPredicateFn::new(projection_mask, move |batch| { + let batch = transformer + .process_record_batch(batch) + .map_err(|err| ArrowError::ExternalError(Box::new(err)))?; + predicate_func(batch) + }); Ok(RowFilter::new(vec![Box::new(arrow_predicate)])) } @@ -227,8 +245,8 @@ mod tests { use arrow_array::cast::AsArray; use arrow_array::types::Int64Type; use arrow_array::{ - ArrayRef, Decimal128Array, Float32Array, Int32Array, Int64Array, LargeStringArray, - RecordBatch, StringArray, + ArrayRef, Decimal128Array, Float32Array, Float64Array, Int32Array, Int64Array, + LargeStringArray, RecordBatch, StringArray, }; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; use futures::TryStreamExt; @@ -463,6 +481,89 @@ mod tests { } } + /// Writes `column` as field 1, reads it back as `iceberg_type` under each predicate, and + /// checks the values that survive (`None` when no row does). + async fn assert_promoted_reads( + column: ArrayRef, + iceberg_type: PrimitiveType, + cases: Vec<(Predicate, Option)>, + ) { + let tmp = TempDir::new().unwrap(); + let path = format!("{}/column.parquet", tmp.path().to_str().unwrap()); + let arrow_schema = Arc::new(ArrowSchema::new(vec![field_with_id( + "v", + column.data_type().clone(), + 1, + )])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![column]).unwrap(); + write_row_groups(&path, arrow_schema, vec![batch], false); + + let schema = value_schema(Type::Primitive(iceberg_type)); + let file_size = std::fs::metadata(&path).unwrap().len(); + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + for (predicate, expected) in cases { + let task = FileScanTask::builder() + .with_file_size_in_bytes(file_size) + .with_start(0) + .with_length(0) + .with_data_file_path(path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1]) + .with_predicate(Some(predicate.bind(schema.clone(), true).unwrap())) + .with_case_sensitive(true) + .build() + .unwrap(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .clone() + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + let actual = collapse(&batches).map(|batch| Arc::clone(batch.column(0))); + assert_eq!(actual, expected, "{predicate}"); + } + } + + /// A file written before a type promotion keeps the old physical type, so the row + /// filter must compare its values promoted to the table type. + #[tokio::test] + async fn test_predicate_on_promoted_column() { + let longs = |values: Vec| Some(Arc::new(Int64Array::from(values)) as ArrayRef); + let all = longs(vec![1, 2]); + let above = || Datum::long(i64::from(i32::MAX) + 1); + let below = || Datum::long(i64::from(i32::MIN) - 1); + let v = || Reference::new("v"); + assert_promoted_reads( + Arc::new(Int32Array::from(vec![1, 2])), + PrimitiveType::Long, + vec![ + (v().less_than(above()), all.clone()), + (v().less_than_or_equal_to(above()), all.clone()), + (v().greater_than_or_equal_to(below()), all.clone()), + (v().not_equal_to(above()), all), + (v().is_in([Datum::long(1), above()]), longs(vec![1])), + (v().is_not_in([Datum::long(1), above()]), longs(vec![2])), + ], + ) + .await; + + // 0.1f32 promotes to a double that is greater than 0.1, not equal to it. + let promoted = Arc::new(Float64Array::from(vec![f64::from(0.1f32)])) as ArrayRef; + assert_promoted_reads( + Arc::new(Float32Array::from(vec![0.1])), + PrimitiveType::Double, + vec![ + (v().greater_than(Datum::double(0.1)), Some(promoted)), + (v().equal_to(Datum::double(0.1)), None), + ], + ) + .await; + } + /// Verifies that file splits respect byte ranges and only read specific row groups. #[tokio::test] async fn test_file_splits_respect_byte_ranges() { From 6e9aba7d119cd0d086ebcf9bf1423850c6324076 Mon Sep 17 00:00:00 2001 From: comphead Date: Wed, 28 Oct 2026 09:54:42 -0700 Subject: [PATCH 2/7] test(reader): cover files without field ids and date promotion; name the strict cast Reuse read_once in assert_promoted_reads, read each file with and without a field id, and add a date -> timestamp case. Explain why try_cast_literal sets safe: false. --- .../src/arrow/reader/predicate_visitor.rs | 1 + crates/iceberg/src/arrow/reader/row_filter.rs | 81 ++++++++++--------- 2 files changed, 44 insertions(+), 38 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/predicate_visitor.rs b/crates/iceberg/src/arrow/reader/predicate_visitor.rs index 45a3d49983..cc42b90f04 100644 --- a/crates/iceberg/src/arrow/reader/predicate_visitor.rs +++ b/crates/iceberg/src/arrow/reader/predicate_visitor.rs @@ -900,6 +900,7 @@ fn try_cast_literal( return Ok(Arc::clone(literal)); } + // `safe: false` makes arrow return an error where the default would produce null. let options = CastOptions { safe: false, ..Default::default() diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs index 253e39b12d..9266627e51 100644 --- a/crates/iceberg/src/arrow/reader/row_filter.rs +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -245,8 +245,8 @@ mod tests { use arrow_array::cast::AsArray; use arrow_array::types::Int64Type; use arrow_array::{ - ArrayRef, Decimal128Array, Float32Array, Float64Array, Int32Array, Int64Array, - LargeStringArray, RecordBatch, StringArray, + ArrayRef, Date32Array, Decimal128Array, Float32Array, Float64Array, Int32Array, Int64Array, + LargeStringArray, RecordBatch, StringArray, TimestampMicrosecondArray, }; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; use futures::TryStreamExt; @@ -481,50 +481,45 @@ mod tests { } } - /// Writes `column` as field 1, reads it back as `iceberg_type` under each predicate, and - /// checks the values that survive (`None` when no row does). + /// Writes `column` as field 1, with and without a field id, reads it back as `iceberg_type` + /// under each predicate, and checks the values that survive (`None` when no row does). async fn assert_promoted_reads( column: ArrayRef, iceberg_type: PrimitiveType, cases: Vec<(Predicate, Option)>, ) { let tmp = TempDir::new().unwrap(); - let path = format!("{}/column.parquet", tmp.path().to_str().unwrap()); - let arrow_schema = Arc::new(ArrowSchema::new(vec![field_with_id( - "v", - column.data_type().clone(), - 1, - )])); - let batch = RecordBatch::try_new(arrow_schema.clone(), vec![column]).unwrap(); - write_row_groups(&path, arrow_schema, vec![batch], false); - let schema = value_schema(Type::Primitive(iceberg_type)); - let file_size = std::fs::metadata(&path).unwrap().len(); let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); - for (predicate, expected) in cases { - let task = FileScanTask::builder() - .with_file_size_in_bytes(file_size) - .with_start(0) - .with_length(0) - .with_data_file_path(path.clone()) - .with_data_file_format(DataFileFormat::Parquet) - .with_schema(schema.clone()) - .with_project_field_ids(vec![1]) - .with_predicate(Some(predicate.bind(schema.clone(), true).unwrap())) - .with_case_sensitive(true) - .build() - .unwrap(); - let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; - let batches: Vec = reader - .clone() - .read(tasks) - .unwrap() - .stream() - .try_collect() - .await - .unwrap(); - let actual = collapse(&batches).map(|batch| Arc::clone(batch.column(0))); - assert_eq!(actual, expected, "{predicate}"); + for with_field_id in [true, false] { + // Without a field id, the reader assigns one by position. + let path = format!("{}/{with_field_id}.parquet", tmp.path().to_str().unwrap()); + let field = if with_field_id { + field_with_id("v", column.data_type().clone(), 1) + } else { + Field::new("v", column.data_type().clone(), false) + }; + let arrow_schema = Arc::new(ArrowSchema::new(vec![field])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![column.clone()]).unwrap(); + write_row_groups(&path, arrow_schema, vec![batch], false); + + for (predicate, expected) in &cases { + let (batches, _) = read_once( + reader.clone(), + &path, + schema.clone(), + vec![1], + predicate.clone(), + None, + vec![], + ) + .await; + let actual = collapse(&batches).map(|batch| Arc::clone(batch.column(0))); + assert_eq!( + &actual, expected, + "{predicate}, with_field_id = {with_field_id}" + ); + } } } @@ -562,6 +557,16 @@ mod tests { ], ) .await; + + // 2024-01-01 promotes to midnight, which is before noon on that day. + let noon = || Datum::timestamp_from_str("2024-01-01T12:00:00").unwrap(); + let midnight = Arc::new(TimestampMicrosecondArray::from(vec![1_704_067_200_000_000])); + assert_promoted_reads( + Arc::new(Date32Array::from(vec![19723])), + PrimitiveType::Timestamp, + vec![(v().less_than(noon()), Some(midnight as ArrayRef))], + ) + .await; } /// Verifies that file splits respect byte ranges and only read specific row groups. From 256dd12522c09731dfaa446a1fd34db96f9b73cf Mon Sep 17 00:00:00 2001 From: comphead Date: Wed, 28 Oct 2026 15:20:19 -0700 Subject: [PATCH 3/7] perf(reader): build the row filter transformer from the filter fields only The transformer converts its whole schema to Arrow on each file's first batch. Give it a schema of the top-level filter fields so that setup no longer grows with the table's width. --- crates/iceberg/src/arrow/reader/pipeline.rs | 2 +- crates/iceberg/src/arrow/reader/row_filter.rs | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs index 540982617b..0e95c74d2d 100644 --- a/crates/iceberg/src/arrow/reader/pipeline.rs +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -580,7 +580,7 @@ impl FileScanTaskReader { record_batch_stream_builder.parquet_schema(), &iceberg_field_ids, &field_id_map, - task.schema_ref(), + task.schema(), )?; record_batch_stream_builder = record_batch_stream_builder.with_row_filter(row_filter); diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs index 9266627e51..5b5cd1f396 100644 --- a/crates/iceberg/src/arrow/reader/row_filter.rs +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -36,7 +36,7 @@ use crate::expr::BoundPredicate; use crate::expr::visitors::bound_predicate_visitor::visit; use crate::expr::visitors::page_index_evaluator::PageIndexEvaluator; use crate::expr::visitors::row_group_metrics_evaluator::RowGroupMetricsEvaluator; -use crate::spec::{Schema, SchemaRef}; +use crate::spec::Schema; use crate::{Error, ErrorKind}; impl ArrowReader { @@ -45,7 +45,7 @@ impl ArrowReader { parquet_schema: &SchemaDescriptor, iceberg_field_ids: &HashSet, field_id_map: &HashMap, - snapshot_schema: SchemaRef, + snapshot_schema: &Schema, ) -> Result { // Collect Parquet column indices from field ids, keeping each column's field id. // If the field id is not found in Parquet schema, it will be ignored due to schema evolution. @@ -73,9 +73,18 @@ impl ArrowReader { let mut predicate_func = visit(&mut converter, predicates)?; // Predicates are bound to the table schema, so evaluate them on the filter columns // converted to it, as the reader's output is (e.g. a file written before an `int` -> - // `long` promotion). + // `long` promotion). The transformer's schema holds only the top-level filter fields, so + // its per-file setup does not grow with the table's width. + let filter_schema = Schema::builder() + .with_schema_id(snapshot_schema.schema_id()) + .with_fields( + field_ids + .iter() + .filter_map(|id| snapshot_schema.as_struct().field_by_id(*id).cloned()), + ) + .build()?; let mut transformer = - RecordBatchTransformerBuilder::new(snapshot_schema, &field_ids).build(); + RecordBatchTransformerBuilder::new(Arc::new(filter_schema), &field_ids).build(); let arrow_predicate = ArrowPredicateFn::new(projection_mask, move |batch| { let batch = transformer .process_record_batch(batch) From 316be2d31d57b9bcb6d3596d58b6eaac64ad75c0 Mon Sep 17 00:00:00 2001 From: comphead Date: Mon, 16 Nov 2026 11:22:21 -0800 Subject: [PATCH 4/7] fix(reader): build the row filter schema from each leaf's top-level field A filter leaf nested in a struct has no top-level field of its own, so the filter schema missed it and the transformer failed with `field not found` once nested predicates reach the row filter (#2906). Take each leaf's top-level field, prune it to the filter leaves, and pass the top-level ids in leaf order. --- crates/iceberg/src/arrow/reader/row_filter.rs | 80 +++++++++++++++---- 1 file changed, 66 insertions(+), 14 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs index 5b5cd1f396..2b1389c32b 100644 --- a/crates/iceberg/src/arrow/reader/row_filter.rs +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -36,7 +36,7 @@ use crate::expr::BoundPredicate; use crate::expr::visitors::bound_predicate_visitor::visit; use crate::expr::visitors::page_index_evaluator::PageIndexEvaluator; use crate::expr::visitors::row_group_metrics_evaluator::RowGroupMetricsEvaluator; -use crate::spec::Schema; +use crate::spec::{NestedFieldRef, Schema, Type, prune_columns}; use crate::{Error, ErrorKind}; impl ArrowReader { @@ -73,18 +73,10 @@ impl ArrowReader { let mut predicate_func = visit(&mut converter, predicates)?; // Predicates are bound to the table schema, so evaluate them on the filter columns // converted to it, as the reader's output is (e.g. a file written before an `int` -> - // `long` promotion). The transformer's schema holds only the top-level filter fields, so - // its per-file setup does not grow with the table's width. - let filter_schema = Schema::builder() - .with_schema_id(snapshot_schema.schema_id()) - .with_fields( - field_ids - .iter() - .filter_map(|id| snapshot_schema.as_struct().field_by_id(*id).cloned()), - ) - .build()?; + // `long` promotion). + let (filter_schema, root_ids) = filter_schema(snapshot_schema, &field_ids)?; let mut transformer = - RecordBatchTransformerBuilder::new(Arc::new(filter_schema), &field_ids).build(); + RecordBatchTransformerBuilder::new(Arc::new(filter_schema), &root_ids).build(); let arrow_predicate = ArrowPredicateFn::new(projection_mask, move |batch| { let batch = transformer .process_record_batch(batch) @@ -245,6 +237,33 @@ impl ArrowReader { } } +/// The top-level fields holding `leaf_ids`, pruned to those leaves, and their ids in leaf order +/// (the filter batch's column order). Only these fields are converted, so the transformer's +/// per-file setup does not grow with the table's width. +fn filter_schema(snapshot_schema: &Schema, leaf_ids: &[i32]) -> Result<(Schema, Vec)> { + let mut roots: Vec = Vec::with_capacity(leaf_ids.len()); + for accessor in leaf_ids + .iter() + .filter_map(|id| snapshot_schema.accessor_by_field_id(*id)) + { + let root = &snapshot_schema.as_struct().fields()[accessor.position()]; + if !roots.iter().any(|r| r.id == root.id) { + roots.push(Arc::clone(root)); + } + } + let root_ids = roots.iter().map(|r| r.id).collect(); + let mut schema = Schema::builder().with_schema_id(snapshot_schema.schema_id()); + // `prune_columns` panics when it selects nothing, e.g. when no filter column is in the file. + if !roots.is_empty() { + let roots = Schema::builder().with_fields(roots).build()?; + let Type::Struct(pruned) = prune_columns(&roots, leaf_ids.iter().copied(), false)? else { + unreachable!("prune_columns returns a struct"); + }; + schema = schema.with_fields(pruned.fields().iter().cloned()); + } + Ok((schema.build()?, root_ids)) +} + #[cfg(test)] mod tests { use std::collections::HashMap; @@ -268,6 +287,7 @@ mod tests { use parquet::schema::types::SchemaDescriptor; use tempfile::TempDir; + use super::filter_schema; use crate::Runtime; use crate::arrow::{ ArrowReader, ArrowReaderBuilder, ArrowReaderOptions as ReaderOptions, ParquetReadOptions, @@ -276,8 +296,8 @@ mod tests { use crate::io::FileIO; use crate::scan::{FileScanTask, FileScanTaskDeleteFile, FileScanTaskStream}; use crate::spec::{ - DataContentType, DataFileFormat, Datum, Literal, NestedField, PartitionSpec, PrimitiveType, - Schema, SchemaRef, Struct, Transform, Type, + DataContentType, DataFileFormat, Datum, Literal, NestedField, NestedFieldRef, + PartitionSpec, PrimitiveType, Schema, SchemaRef, Struct, StructType, Transform, Type, }; async fn test_perform_read( @@ -532,6 +552,38 @@ mod tests { } } + /// The filter schema holds each filter leaf's top-level field, pruned to the filter leaves, + /// with root ids in leaf order. + #[test] + fn test_filter_schema_prunes_nested_roots() { + let person = |fields: Vec| { + NestedField::required(2, "person", Type::Struct(StructType::new(fields))).into() + }; + let n = || NestedField::required(3, "n", Type::Primitive(PrimitiveType::Long)).into(); + let m = || NestedField::optional(4, "m", Type::Primitive(PrimitiveType::Int)).into(); + let id = || NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(); + let schema = Schema::builder() + .with_fields(vec![id(), person(vec![n(), m()])]) + .build() + .unwrap(); + + for (leaf_ids, expected_fields, expected_root_ids) in [ + (vec![3], vec![person(vec![n()])], vec![2]), + (vec![4, 1, 3], vec![person(vec![n(), m()]), id()], vec![ + 2, 1, + ]), + (vec![], vec![], vec![]), + ] { + let (filter_schema, root_ids) = filter_schema(&schema, &leaf_ids).unwrap(); + assert_eq!( + filter_schema.as_struct().fields(), + expected_fields, + "{leaf_ids:?}" + ); + assert_eq!(root_ids, expected_root_ids, "{leaf_ids:?}"); + } + } + /// A file written before a type promotion keeps the old physical type, so the row /// filter must compare its values promoted to the table type. #[tokio::test] From 87900740dd9168178b39c1903d54f4e8a8c8180d Mon Sep 17 00:00:00 2001 From: comphead Date: Sat, 21 Nov 2026 10:59:15 -0800 Subject: [PATCH 5/7] fix(reader): reject a filter leaf that is not in the scan schema filter_schema skipped a leaf without an accessor in the scan schema, so the converted filter batch had fewer columns than the predicate's column positions, and the predicate panicked with an index out of bounds. Only a task whose predicate was bound to another schema gets there. Fail with DataInvalid instead, as validate_equality_ids does for equality deletes and the row group evaluator does with default options. --- crates/iceberg/src/arrow/reader/row_filter.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs index 2b1389c32b..899a388af8 100644 --- a/crates/iceberg/src/arrow/reader/row_filter.rs +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -239,13 +239,17 @@ impl ArrowReader { /// The top-level fields holding `leaf_ids`, pruned to those leaves, and their ids in leaf order /// (the filter batch's column order). Only these fields are converted, so the transformer's -/// per-file setup does not grow with the table's width. +/// per-file setup does not grow with the table's width. Fails for a leaf that is not in +/// `snapshot_schema`, because skipping it would shift the columns the predicate reads. fn filter_schema(snapshot_schema: &Schema, leaf_ids: &[i32]) -> Result<(Schema, Vec)> { let mut roots: Vec = Vec::with_capacity(leaf_ids.len()); - for accessor in leaf_ids - .iter() - .filter_map(|id| snapshot_schema.accessor_by_field_id(*id)) - { + for &id in leaf_ids { + let accessor = snapshot_schema.accessor_by_field_id(id).ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!("Predicate field id {id} is not in the scan schema"), + ) + })?; let root = &snapshot_schema.as_struct().fields()[accessor.position()]; if !roots.iter().any(|r| r.id == root.id) { roots.push(Arc::clone(root)); @@ -553,7 +557,7 @@ mod tests { } /// The filter schema holds each filter leaf's top-level field, pruned to the filter leaves, - /// with root ids in leaf order. + /// with root ids in leaf order. A leaf missing from the schema fails. #[test] fn test_filter_schema_prunes_nested_roots() { let person = |fields: Vec| { @@ -582,6 +586,7 @@ mod tests { ); assert_eq!(root_ids, expected_root_ids, "{leaf_ids:?}"); } + assert!(filter_schema(&schema, &[3, 9]).is_err()); } /// A file written before a type promotion keeps the old physical type, so the row From 762207790b260470825b14c0148c3ffba18b66f9 Mon Sep 17 00:00:00 2001 From: comphead Date: Sat, 21 Nov 2026 11:26:01 -0800 Subject: [PATCH 6/7] fix(spec): return an empty struct from prune_columns for an empty selection PruneColumn::schema unwrapped the visited struct, so selecting nothing panicked before prune_columns reached its fallback to an empty struct. Return the value as is, so the fallback applies. --- crates/iceberg/src/spec/schema/prune_columns.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/iceberg/src/spec/schema/prune_columns.rs b/crates/iceberg/src/spec/schema/prune_columns.rs index edfbb5fa1d..a3b56f27c7 100644 --- a/crates/iceberg/src/spec/schema/prune_columns.rs +++ b/crates/iceberg/src/spec/schema/prune_columns.rs @@ -104,7 +104,7 @@ impl SchemaVisitor for PruneColumn { type T = Option; fn schema(&mut self, _schema: &Schema, value: Option) -> Result> { - Ok(Some(value.unwrap())) + Ok(value) } fn field(&mut self, field: &NestedFieldRef, value: Option) -> Result> { @@ -745,6 +745,9 @@ mod tests { let result = prune_columns(&schema, selected, true); assert!(result.is_ok()); assert_eq!(result.unwrap(), Type::Struct(schema.as_struct().clone())); + // Selecting nothing gives an empty struct. + let empty = prune_columns(&schema, [], false).unwrap(); + assert_eq!(empty, Type::Struct(StructType::default())); } #[test] From 0d93d0d4c391a2a9212a07a3bc33eaef7e8f55fd Mon Sep 17 00:00:00 2001 From: comphead Date: Sat, 21 Nov 2026 11:26:01 -0800 Subject: [PATCH 7/7] refactor(reader): simplify the row filter schema and its tests - filter_schema returns only the schema. The transformer's root ids are its top-level fields, and the empty-selection guard is gone now that prune_columns handles it. - Use invalid_data! for the unknown-leaf error, and drop the unused schema id. - try_cast_literal's doc no longer cites Binary and LargeBinary, which now map to one Arrow type. - test_predicate_converter_rejects_lossy_literal_cast reuses apply_predicate_to_batch, which now takes the file's message type and returns the predicate's result. - test_predicate_on_promoted_column keeps one case per promotion. The dropped cases passed or failed together with the kept ones under every mutation of the row filter. --- .../src/arrow/reader/predicate_visitor.rs | 71 ++++----- crates/iceberg/src/arrow/reader/row_filter.rs | 139 ++++++++---------- 2 files changed, 87 insertions(+), 123 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/predicate_visitor.rs b/crates/iceberg/src/arrow/reader/predicate_visitor.rs index cc42b90f04..2449947b2a 100644 --- a/crates/iceberg/src/arrow/reader/predicate_visitor.rs +++ b/crates/iceberg/src/arrow/reader/predicate_visitor.rs @@ -885,10 +885,9 @@ impl BoundPredicateVisitor for PredicateConverter<'_> { } } -/// The row filter evaluates batches converted to the table schema, but a literal can still use -/// another Arrow representation of the same type (e.g. `Binary` for a `LargeBinary` column). -/// The compute kernels need the exact type, so cast the literal, failing rather than letting a -/// lossy cast turn it into null. +/// The row filter converts its batches to the table schema, so a literal already has the +/// column's Arrow type unless the two differ in representation only. Cast it in that case, +/// failing rather than letting a lossy cast turn it into null. fn try_cast_literal( literal: &Arc, column_type: &DataType, @@ -914,7 +913,7 @@ mod tests { use std::collections::{HashMap, HashSet}; use std::sync::Arc; - use arrow_array::{Array, BooleanArray, Int32Array, RecordBatch}; + use arrow_array::{Array, ArrayRef, BooleanArray, Int32Array, RecordBatch}; use arrow_schema::{ArrowError, DataType, Field, Schema as ArrowSchema}; use parquet::schema::parser::parse_message_type; use parquet::schema::types::SchemaDescriptor; @@ -922,6 +921,7 @@ mod tests { use super::{ CollectFieldIdVisitor, PredicateConverter, constant_bool_array, residual_for_missing_fields, }; + use crate::arrow::reader::projection::build_field_id_map; use crate::expr::visitors::bound_predicate_visitor::visit; use crate::expr::{Bind, BoundPredicate, Predicate, Reference}; use crate::spec::{ @@ -1017,24 +1017,19 @@ mod tests { } } + /// Evaluates `predicate`, bound to `schema`, on `batch` read from a Parquet file of + /// `message_type`. fn apply_predicate_to_batch( predicate: Predicate, schema: SchemaRef, + message_type: &str, batch: RecordBatch, - ) -> BooleanArray { + ) -> Result { let bound = predicate.bind(schema, true).unwrap(); - - // Build a trivial Parquet schema with one float column at field id 4 - let message_type = " - message schema { - optional float qux = 4; - } - "; let parquet_type = parse_message_type(message_type).expect("parse schema"); let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_type)); - - let column_map = HashMap::from([(4i32, 0usize)]); - let column_indices = vec![0usize]; + let column_map = build_field_id_map(&parquet_schema).unwrap().unwrap(); + let column_indices: Vec = (0..parquet_schema.num_columns()).collect(); let mut converter = PredicateConverter { parquet_schema: &parquet_schema, @@ -1043,13 +1038,14 @@ mod tests { }; let mut predicate_fn = visit(&mut converter, &bound).unwrap(); - predicate_fn(batch).unwrap() + predicate_fn(batch) } #[test] fn test_predicate_converter_nan() { use arrow_array::Float32Array; + const QUX: &str = "message schema { optional float qux = 4; }"; let schema = table_schema_simple(); let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( "qux", @@ -1064,7 +1060,8 @@ mod tests { ))]) .unwrap(); let result = - apply_predicate_to_batch(Reference::new("qux").is_nan(), schema.clone(), batch); + apply_predicate_to_batch(Reference::new("qux").is_nan(), schema.clone(), QUX, batch) + .unwrap(); assert_eq!( [ result.value(0), @@ -1079,7 +1076,9 @@ mod tests { // not_nan: non-null-propagating per Java's implementation - NULL → true let batch = RecordBatch::try_new(arrow_schema, vec![Arc::new(Float32Array::from(values))]).unwrap(); - let result = apply_predicate_to_batch(Reference::new("qux").is_not_nan(), schema, batch); + let result = + apply_predicate_to_batch(Reference::new("qux").is_not_nan(), schema, QUX, batch) + .unwrap(); assert_eq!( [ result.value(0), @@ -1352,31 +1351,15 @@ mod tests { .build() .unwrap(), ); - let predicate = Reference::new("x") - .less_than(Datum::long(i64::from(i32::MAX) + 1)) - .bind(schema, true) - .unwrap(); - let parquet_type = parse_message_type("message schema { required int32 x = 1; }").unwrap(); - let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_type)); - let column_map = HashMap::from([(1, 0)]); - let column_indices = vec![0]; - let mut converter = PredicateConverter { - parquet_schema: &parquet_schema, - column_map: &column_map, - column_indices: &column_indices, - }; - let mut predicate_fn = visit(&mut converter, &predicate).unwrap(); - // An `int` batch that was not converted to the table's `long`. - let batch = RecordBatch::try_new( - Arc::new(ArrowSchema::new(vec![Field::new( - "x", - DataType::Int32, - false, - )])), - vec![Arc::new(Int32Array::from(vec![1]))], - ) - .unwrap(); - assert!(matches!(predicate_fn(batch), Err(ArrowError::CastError(_)))); + let column = Arc::new(Int32Array::from(vec![1])) as ArrayRef; + let batch = RecordBatch::try_from_iter([("x", column)]).unwrap(); + let result = apply_predicate_to_batch( + Reference::new("x").less_than(Datum::long(i64::from(i32::MAX) + 1)), + schema, + "message schema { required int32 x = 1; }", + batch, + ); + assert!(matches!(result, Err(ArrowError::CastError(_)))); } } diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs index 899a388af8..e29b994984 100644 --- a/crates/iceberg/src/arrow/reader/row_filter.rs +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -31,7 +31,7 @@ use parquet::schema::types::SchemaDescriptor; use super::{ArrowReader, PredicateConverter}; use crate::arrow::record_batch_transformer::RecordBatchTransformerBuilder; -use crate::error::Result; +use crate::error::{Result, invalid_data}; use crate::expr::BoundPredicate; use crate::expr::visitors::bound_predicate_visitor::visit; use crate::expr::visitors::page_index_evaluator::PageIndexEvaluator; @@ -71,10 +71,15 @@ impl ArrowReader { // creates the projection mask for the Arrow predicates. let projection_mask = ProjectionMask::leaves(parquet_schema, column_indices.clone()); let mut predicate_func = visit(&mut converter, predicates)?; - // Predicates are bound to the table schema, so evaluate them on the filter columns - // converted to it, as the reader's output is (e.g. a file written before an `int` -> - // `long` promotion). - let (filter_schema, root_ids) = filter_schema(snapshot_schema, &field_ids)?; + // Predicates are bound to the table schema, so convert the filter columns to it, as the + // reader does for its output (e.g. `int` read as `long`). + let filter_schema = filter_schema(snapshot_schema, &field_ids)?; + let root_ids: Vec = filter_schema + .as_struct() + .fields() + .iter() + .map(|f| f.id) + .collect(); let mut transformer = RecordBatchTransformerBuilder::new(Arc::new(filter_schema), &root_ids).build(); let arrow_predicate = ArrowPredicateFn::new(projection_mask, move |batch| { @@ -237,35 +242,28 @@ impl ArrowReader { } } -/// The top-level fields holding `leaf_ids`, pruned to those leaves, and their ids in leaf order -/// (the filter batch's column order). Only these fields are converted, so the transformer's -/// per-file setup does not grow with the table's width. Fails for a leaf that is not in -/// `snapshot_schema`, because skipping it would shift the columns the predicate reads. -fn filter_schema(snapshot_schema: &Schema, leaf_ids: &[i32]) -> Result<(Schema, Vec)> { +/// The top-level fields holding `leaf_ids`, pruned to those leaves, in leaf order (the filter +/// batch's column order). Only these fields are converted, so the transformer's per-file setup +/// does not grow with the table's width. Fails for a leaf that is not in `snapshot_schema`, +/// because skipping it would shift the columns the predicate reads. +fn filter_schema(snapshot_schema: &Schema, leaf_ids: &[i32]) -> Result { let mut roots: Vec = Vec::with_capacity(leaf_ids.len()); for &id in leaf_ids { - let accessor = snapshot_schema.accessor_by_field_id(id).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Predicate field id {id} is not in the scan schema"), - ) - })?; + let accessor = snapshot_schema + .accessor_by_field_id(id) + .ok_or_else(|| invalid_data!("Predicate field id {id} is not in the scan schema"))?; let root = &snapshot_schema.as_struct().fields()[accessor.position()]; if !roots.iter().any(|r| r.id == root.id) { roots.push(Arc::clone(root)); } } - let root_ids = roots.iter().map(|r| r.id).collect(); - let mut schema = Schema::builder().with_schema_id(snapshot_schema.schema_id()); - // `prune_columns` panics when it selects nothing, e.g. when no filter column is in the file. - if !roots.is_empty() { - let roots = Schema::builder().with_fields(roots).build()?; - let Type::Struct(pruned) = prune_columns(&roots, leaf_ids.iter().copied(), false)? else { - unreachable!("prune_columns returns a struct"); - }; - schema = schema.with_fields(pruned.fields().iter().cloned()); - } - Ok((schema.build()?, root_ids)) + let roots = Schema::builder().with_fields(roots).build()?; + let Type::Struct(pruned) = prune_columns(&roots, leaf_ids.iter().copied(), false)? else { + unreachable!("prune_columns returns a struct"); + }; + Schema::builder() + .with_fields(pruned.fields().to_vec()) + .build() } #[cfg(test)] @@ -277,7 +275,7 @@ mod tests { use arrow_array::cast::AsArray; use arrow_array::types::Int64Type; use arrow_array::{ - ArrayRef, Date32Array, Decimal128Array, Float32Array, Float64Array, Int32Array, Int64Array, + ArrayRef, Date32Array, Decimal128Array, Float32Array, Int32Array, Int64Array, LargeStringArray, RecordBatch, StringArray, TimestampMicrosecondArray, }; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; @@ -515,11 +513,12 @@ mod tests { } /// Writes `column` as field 1, with and without a field id, reads it back as `iceberg_type` - /// under each predicate, and checks the values that survive (`None` when no row does). + /// under `predicate`, and checks the values that survive (`None` when no row does). async fn assert_promoted_reads( column: ArrayRef, iceberg_type: PrimitiveType, - cases: Vec<(Predicate, Option)>, + predicate: Predicate, + expected: Option, ) { let tmp = TempDir::new().unwrap(); let schema = value_schema(Type::Primitive(iceberg_type)); @@ -536,28 +535,26 @@ mod tests { let batch = RecordBatch::try_new(arrow_schema.clone(), vec![column.clone()]).unwrap(); write_row_groups(&path, arrow_schema, vec![batch], false); - for (predicate, expected) in &cases { - let (batches, _) = read_once( - reader.clone(), - &path, - schema.clone(), - vec![1], - predicate.clone(), - None, - vec![], - ) - .await; - let actual = collapse(&batches).map(|batch| Arc::clone(batch.column(0))); - assert_eq!( - &actual, expected, - "{predicate}, with_field_id = {with_field_id}" - ); - } + let (batches, _) = read_once( + reader.clone(), + &path, + schema.clone(), + vec![1], + predicate.clone(), + None, + vec![], + ) + .await; + let actual = collapse(&batches).map(|batch| Arc::clone(batch.column(0))); + assert_eq!( + actual, expected, + "{predicate}, with_field_id = {with_field_id}" + ); } } /// The filter schema holds each filter leaf's top-level field, pruned to the filter leaves, - /// with root ids in leaf order. A leaf missing from the schema fails. + /// in leaf order. A leaf missing from the schema fails. #[test] fn test_filter_schema_prunes_nested_roots() { let person = |fields: Vec| { @@ -571,20 +568,17 @@ mod tests { .build() .unwrap(); - for (leaf_ids, expected_fields, expected_root_ids) in [ - (vec![3], vec![person(vec![n()])], vec![2]), - (vec![4, 1, 3], vec![person(vec![n(), m()]), id()], vec![ - 2, 1, - ]), - (vec![], vec![], vec![]), + for (leaf_ids, expected_fields) in [ + (vec![3], vec![person(vec![n()])]), + (vec![4, 1, 3], vec![person(vec![n(), m()]), id()]), + (vec![], vec![]), ] { - let (filter_schema, root_ids) = filter_schema(&schema, &leaf_ids).unwrap(); + let filter_schema = filter_schema(&schema, &leaf_ids).unwrap(); assert_eq!( filter_schema.as_struct().fields(), expected_fields, "{leaf_ids:?}" ); - assert_eq!(root_ids, expected_root_ids, "{leaf_ids:?}"); } assert!(filter_schema(&schema, &[3, 9]).is_err()); } @@ -593,44 +587,31 @@ mod tests { /// filter must compare its values promoted to the table type. #[tokio::test] async fn test_predicate_on_promoted_column() { - let longs = |values: Vec| Some(Arc::new(Int64Array::from(values)) as ArrayRef); - let all = longs(vec![1, 2]); - let above = || Datum::long(i64::from(i32::MAX) + 1); - let below = || Datum::long(i64::from(i32::MIN) - 1); let v = || Reference::new("v"); + // The literal is out of `int` range. assert_promoted_reads( Arc::new(Int32Array::from(vec![1, 2])), PrimitiveType::Long, - vec![ - (v().less_than(above()), all.clone()), - (v().less_than_or_equal_to(above()), all.clone()), - (v().greater_than_or_equal_to(below()), all.clone()), - (v().not_equal_to(above()), all), - (v().is_in([Datum::long(1), above()]), longs(vec![1])), - (v().is_not_in([Datum::long(1), above()]), longs(vec![2])), - ], + v().less_than(Datum::long(i64::from(i32::MAX) + 1)), + Some(Arc::new(Int64Array::from(vec![1, 2]))), ) .await; - - // 0.1f32 promotes to a double that is greater than 0.1, not equal to it. - let promoted = Arc::new(Float64Array::from(vec![f64::from(0.1f32)])) as ArrayRef; + // 0.1f32 promotes to a double slightly above 0.1, so it no longer equals 0.1. assert_promoted_reads( Arc::new(Float32Array::from(vec![0.1])), PrimitiveType::Double, - vec![ - (v().greater_than(Datum::double(0.1)), Some(promoted)), - (v().equal_to(Datum::double(0.1)), None), - ], + v().equal_to(Datum::double(0.1)), + None, ) .await; - // 2024-01-01 promotes to midnight, which is before noon on that day. - let noon = || Datum::timestamp_from_str("2024-01-01T12:00:00").unwrap(); - let midnight = Arc::new(TimestampMicrosecondArray::from(vec![1_704_067_200_000_000])); assert_promoted_reads( Arc::new(Date32Array::from(vec![19723])), PrimitiveType::Timestamp, - vec![(v().less_than(noon()), Some(midnight as ArrayRef))], + v().less_than(Datum::timestamp_from_str("2024-01-01T12:00:00").unwrap()), + Some(Arc::new(TimestampMicrosecondArray::from(vec![ + 1_704_067_200_000_000, + ]))), ) .await; }