From d3740b3c2c3ade653d13ef0b11d25fa0c86dff16 Mon Sep 17 00:00:00 2001 From: Gabor Kaszab Date: Tue, 29 Sep 2026 09:47:24 +0200 Subject: [PATCH 1/9] Core, Spark 4.2: Wire in InputFilesDecryptor into Spark readers --- .../encryption/InputFilesDecryptor.java | 5 ++++ .../iceberg/spark/source/BaseReader.java | 30 ++++++++----------- .../iceberg/spark/source/BatchDataReader.java | 9 ++---- .../spark/source/ChangelogRowReader.java | 11 +++++-- .../source/PositionDeletesRowReader.java | 13 ++++---- .../iceberg/spark/source/RowDataReader.java | 10 ++----- .../iceberg/spark/source/TestBaseReader.java | 7 ++--- 7 files changed, 42 insertions(+), 43 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java b/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java index 6245fa119191..935e03be0831 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java +++ b/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java @@ -53,6 +53,11 @@ public static InputFilesDecryptor fromTasks( return new InputFilesDecryptor(() -> referencedFiles(tasks).iterator(), encryptingIO); } + public static InputFilesDecryptor fromFiles( + Iterable> files, EncryptingFileIO encryptingIO) { + return new InputFilesDecryptor(files, encryptingIO); + } + private InputFilesDecryptor( Iterable> files, EncryptingFileIO encryptingIO) { this.referencedFiles = files; diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java index a4d9766ae713..ca43a81523d0 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java @@ -27,8 +27,6 @@ import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.iceberg.ContentFile; import org.apache.iceberg.ContentScanTask; import org.apache.iceberg.DeleteFile; import org.apache.iceberg.MetadataColumns; @@ -44,6 +42,7 @@ import org.apache.iceberg.data.DeleteLoader; import org.apache.iceberg.deletes.DeleteCounter; import org.apache.iceberg.encryption.EncryptingFileIO; +import org.apache.iceberg.encryption.InputFilesDecryptor; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; @@ -78,7 +77,7 @@ abstract class BaseReader implements Closeable { private final DeleteCounter counter; private final boolean cacheDeleteFilesOnExecutors; - private Map lazyInputFiles; + private final InputFilesDecryptor inputFilesDecryptor; private CloseableIterator currentIterator; private T current = null; private TaskT currentTask = null; @@ -102,11 +101,12 @@ abstract class BaseReader implements Closeable { nameMappingString != null ? NameMappingParser.fromJson(nameMappingString) : null; this.counter = new DeleteCounter(); this.cacheDeleteFilesOnExecutors = cacheDeleteFilesOnExecutors; + this.inputFilesDecryptor = newInputFilesDecryptor(); } protected abstract CloseableIterator open(TaskT task); - protected abstract Stream> referencedFiles(TaskT task); + protected abstract InputFilesDecryptor newInputFilesDecryptor(); protected Schema expectedSchema() { return expectedSchema; @@ -148,12 +148,8 @@ public boolean next() throws IOException { } } } catch (IOException | RuntimeException e) { - if (currentTask != null && !currentTask.isDataTask()) { - String filePaths = - referencedFiles(currentTask) - .map(ContentFile::location) - .collect(Collectors.joining(", ")); - LOG.error("Error reading file(s): {}", filePaths, e); + if (currentTask instanceof ContentScanTask contentTask && !currentTask.isDataTask()) { + LOG.error("Error reading file: {}", contentTask.file().location(), e); } throw e; } @@ -177,17 +173,15 @@ public void close() throws IOException { } protected InputFile getInputFile(String location) { - return inputFiles().get(location); + return inputFilesDecryptor.getInputFile(location); } - private Map inputFiles() { - if (lazyInputFiles == null) { - this.lazyInputFiles = - fileIO.bulkDecrypt( - () -> taskGroup.tasks().stream().flatMap(this::referencedFiles).iterator()); - } + protected ScanTaskGroup taskGroup() { + return taskGroup; + } - return lazyInputFiles; + protected EncryptingFileIO encryptingFileIO() { + return fileIO; } protected Map constantsMap(ContentScanTask task, Schema readSchema) { diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BatchDataReader.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BatchDataReader.java index 237dfd5c6919..7c7a3242fd2c 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BatchDataReader.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BatchDataReader.java @@ -19,16 +19,14 @@ package org.apache.iceberg.spark.source; import java.util.Map; -import java.util.stream.Stream; -import org.apache.iceberg.ContentFile; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.ScanTaskGroup; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; +import org.apache.iceberg.encryption.InputFilesDecryptor; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.spark.OrcBatchReadConf; import org.apache.iceberg.spark.ParquetBatchReadConf; import org.apache.iceberg.spark.source.metrics.TaskNumDeletes; @@ -93,8 +91,8 @@ public CustomTaskMetric[] currentMetricsValues() { } @Override - protected Stream> referencedFiles(FileScanTask task) { - return Stream.concat(Stream.of(task.file()), task.deletes().stream()); + protected InputFilesDecryptor newInputFilesDecryptor() { + return InputFilesDecryptor.fromTasks(taskGroup().tasks(), encryptingFileIO()); } @Override @@ -106,7 +104,6 @@ protected CloseableIterator open(FileScanTask task) { InputFileBlockHolder.set(filePath, task.start(), task.length()); InputFile inputFile = getInputFile(filePath); - Preconditions.checkNotNull(inputFile, "Could not find InputFile associated with FileScanTask"); SparkDeleteFilter deleteFilter = new SparkDeleteFilter(filePath, task.deletes(), counter(), true); diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/ChangelogRowReader.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/ChangelogRowReader.java index eb8e5e63f430..bd5e95aa8cbf 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/ChangelogRowReader.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/ChangelogRowReader.java @@ -33,11 +33,11 @@ import org.apache.iceberg.ScanTaskGroup; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; +import org.apache.iceberg.encryption.InputFilesDecryptor; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.spark.rdd.InputFileBlockHolder; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.catalyst.expressions.GenericInternalRow; @@ -134,7 +134,6 @@ private CloseableIterable rows(ContentScanTask task, Sche InputFileBlockHolder.set(filePath, task.start(), task.length()); InputFile location = getInputFile(filePath); - Preconditions.checkNotNull(location, "Could not find InputFile"); return newIterable( location, task.file().format(), @@ -146,7 +145,13 @@ private CloseableIterable rows(ContentScanTask task, Sche } @Override - protected Stream> referencedFiles(ChangelogScanTask task) { + protected InputFilesDecryptor newInputFilesDecryptor() { + return InputFilesDecryptor.fromFiles( + () -> taskGroup().tasks().stream().flatMap(ChangelogRowReader::referencedFiles).iterator(), + encryptingFileIO()); + } + + private static Stream> referencedFiles(ChangelogScanTask task) { if (task instanceof AddedRowsScanTask) { return addedRowsScanTaskFiles((AddedRowsScanTask) task); diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/PositionDeletesRowReader.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/PositionDeletesRowReader.java index c7abf219db02..b11a16dcf079 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/PositionDeletesRowReader.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/PositionDeletesRowReader.java @@ -19,18 +19,17 @@ package org.apache.iceberg.spark.source; import java.util.Map; -import java.util.stream.Stream; import org.apache.iceberg.ContentFile; import org.apache.iceberg.PositionDeletesScanTask; import org.apache.iceberg.ScanTaskGroup; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; +import org.apache.iceberg.encryption.InputFilesDecryptor; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.ExpressionUtil; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.util.ContentFileUtil; import org.apache.spark.rdd.InputFileBlockHolder; import org.apache.spark.sql.catalyst.InternalRow; @@ -67,8 +66,13 @@ class PositionDeletesRowReader extends BaseRowReader } @Override - protected Stream> referencedFiles(PositionDeletesScanTask task) { - return Stream.of(task.file()); + protected InputFilesDecryptor newInputFilesDecryptor() { + return InputFilesDecryptor.fromFiles( + () -> + taskGroup().tasks().stream() + .>map(PositionDeletesScanTask::file) + .iterator(), + encryptingFileIO()); } @SuppressWarnings("resource") // handled by BaseReader @@ -81,7 +85,6 @@ protected CloseableIterator open(PositionDeletesScanTask task) { InputFileBlockHolder.set(filePath, task.start(), task.length()); InputFile inputFile = getInputFile(task.file().location()); - Preconditions.checkNotNull(inputFile, "Could not find InputFile associated with %s", task); // Retain predicates on non-constant fields for row reader filter Map idToConstant = constantsMap(task, expectedSchema()); diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/RowDataReader.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/RowDataReader.java index dbfb0b7614e4..aacbe282f2c7 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/RowDataReader.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/RowDataReader.java @@ -19,18 +19,16 @@ package org.apache.iceberg.spark.source; import java.util.Map; -import java.util.stream.Stream; -import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataTask; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.ScanTaskGroup; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; +import org.apache.iceberg.encryption.InputFilesDecryptor; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.spark.source.metrics.TaskNumDeletes; import org.apache.iceberg.spark.source.metrics.TaskNumSplits; import org.apache.spark.rdd.InputFileBlockHolder; @@ -77,8 +75,8 @@ public CustomTaskMetric[] currentMetricsValues() { } @Override - protected Stream> referencedFiles(FileScanTask task) { - return Stream.concat(Stream.of(task.file()), task.deletes().stream()); + protected InputFilesDecryptor newInputFilesDecryptor() { + return InputFilesDecryptor.fromTasks(taskGroup().tasks(), encryptingFileIO()); } @Override @@ -104,8 +102,6 @@ protected CloseableIterable open( return newDataIterable(task.asDataTask(), readSchema); } else { InputFile inputFile = getInputFile(task.file().location()); - Preconditions.checkNotNull( - inputFile, "Could not find InputFile associated with FileScanTask"); return newIterable( inputFile, task.file().format(), diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestBaseReader.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestBaseReader.java index 0eb9bbe52ffa..f5d0fef3d70f 100644 --- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestBaseReader.java +++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestBaseReader.java @@ -31,18 +31,17 @@ import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.IntStream; -import java.util.stream.Stream; import java.util.stream.StreamSupport; import org.apache.avro.generic.GenericData; import org.apache.iceberg.AppendFiles; import org.apache.iceberg.BaseCombinedScanTask; -import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; +import org.apache.iceberg.encryption.InputFilesDecryptor; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.FileAppender; import org.apache.iceberg.parquet.Parquet; @@ -94,8 +93,8 @@ private static class ClosureTrackingReader extends BaseReader> referencedFiles(FileScanTask task) { - return Stream.of(); + protected InputFilesDecryptor newInputFilesDecryptor() { + return InputFilesDecryptor.fromTasks(taskGroup().tasks(), encryptingFileIO()); } @Override From 3d19f32cd8272d95c8ad9ac179b0784d2f2c3f6b Mon Sep 17 00:00:00 2001 From: Gabor Kaszab Date: Tue, 20 Oct 2026 11:04:28 +0200 Subject: [PATCH 2/9] Move reference files from changelog and pos delete scan into decryptor --- .../encryption/InputFilesDecryptor.java | 51 +++++++++++++++++-- .../spark/source/ChangelogRowReader.java | 36 +------------ .../source/PositionDeletesRowReader.java | 8 +-- 3 files changed, 50 insertions(+), 45 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java b/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java index 935e03be0831..6c0b40ddab63 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java +++ b/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java @@ -20,10 +20,16 @@ import java.util.Collection; import java.util.Map; +import java.util.stream.Stream; +import org.apache.iceberg.AddedRowsScanTask; +import org.apache.iceberg.ChangelogScanTask; import org.apache.iceberg.CombinedScanTask; import org.apache.iceberg.ContentFile; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.DeletedDataFileScanTask; +import org.apache.iceberg.DeletedRowsScanTask; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PositionDeletesScanTask; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; @@ -53,9 +59,15 @@ public static InputFilesDecryptor fromTasks( return new InputFilesDecryptor(() -> referencedFiles(tasks).iterator(), encryptingIO); } - public static InputFilesDecryptor fromFiles( - Iterable> files, EncryptingFileIO encryptingIO) { - return new InputFilesDecryptor(files, encryptingIO); + public static InputFilesDecryptor fromChangelogTasks( + Iterable tasks, EncryptingFileIO encryptingIO) { + return new InputFilesDecryptor(() -> changelogReferencedFiles(tasks).iterator(), encryptingIO); + } + + public static InputFilesDecryptor fromPositionDeletesTasks( + Iterable tasks, EncryptingFileIO encryptingIO) { + return new InputFilesDecryptor( + () -> positionDeletesReferencedFiles(tasks).iterator(), encryptingIO); } private InputFilesDecryptor( @@ -88,6 +100,39 @@ private static Collection> referencedFiles(Iterable return files.values(); } + private static Collection> changelogReferencedFiles( + Iterable tasks) { + Map> files = Maps.newHashMap(); + for (ChangelogScanTask task : tasks) { + if (task instanceof AddedRowsScanTask addedRowsTask) { + Stream.concat(Stream.of(addedRowsTask.file()), addedRowsTask.deletes().stream()) + .forEach(file -> files.put(file.location(), file)); + } else if (task instanceof DeletedDataFileScanTask deletedDataFileTask) { + Stream.concat( + Stream.of(deletedDataFileTask.file()), + deletedDataFileTask.existingDeletes().stream()) + .forEach(file -> files.put(file.location(), file)); + } else if (task instanceof DeletedRowsScanTask) { + throw new UnsupportedOperationException("Deleted rows scan task is not supported yet"); + } else { + throw new IllegalArgumentException( + "Unsupported changelog scan task type: " + task.getClass().getName()); + } + } + + return files.values(); + } + + private static Collection> positionDeletesReferencedFiles( + Iterable tasks) { + Map> files = Maps.newHashMap(); + for (PositionDeletesScanTask task : tasks) { + files.put(task.file().location(), task.file()); + } + + return files.values(); + } + /** * @deprecated since 1.12.0, will be removed in 1.13.0; use {@link #getInputFile(String)} instead. */ diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/ChangelogRowReader.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/ChangelogRowReader.java index bd5e95aa8cbf..03e57d6d834f 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/ChangelogRowReader.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/ChangelogRowReader.java @@ -18,16 +18,12 @@ */ package org.apache.iceberg.spark.source; -import java.util.List; import java.util.Map; -import java.util.stream.Stream; import org.apache.iceberg.AddedRowsScanTask; import org.apache.iceberg.ChangelogScanTask; import org.apache.iceberg.ChangelogUtil; -import org.apache.iceberg.ContentFile; import org.apache.iceberg.ContentScanTask; import org.apache.iceberg.DataFile; -import org.apache.iceberg.DeleteFile; import org.apache.iceberg.DeletedDataFileScanTask; import org.apache.iceberg.DeletedRowsScanTask; import org.apache.iceberg.ScanTaskGroup; @@ -146,36 +142,6 @@ private CloseableIterable rows(ContentScanTask task, Sche @Override protected InputFilesDecryptor newInputFilesDecryptor() { - return InputFilesDecryptor.fromFiles( - () -> taskGroup().tasks().stream().flatMap(ChangelogRowReader::referencedFiles).iterator(), - encryptingFileIO()); - } - - private static Stream> referencedFiles(ChangelogScanTask task) { - if (task instanceof AddedRowsScanTask) { - return addedRowsScanTaskFiles((AddedRowsScanTask) task); - - } else if (task instanceof DeletedRowsScanTask) { - throw new UnsupportedOperationException("Deleted rows scan task is not supported yet"); - - } else if (task instanceof DeletedDataFileScanTask) { - return deletedDataFileScanTaskFiles((DeletedDataFileScanTask) task); - - } else { - throw new IllegalArgumentException( - "Unsupported changelog scan task type: " + task.getClass().getName()); - } - } - - private static Stream> deletedDataFileScanTaskFiles(DeletedDataFileScanTask task) { - DataFile file = task.file(); - List existingDeletes = task.existingDeletes(); - return Stream.concat(Stream.of(file), existingDeletes.stream()); - } - - private static Stream> addedRowsScanTaskFiles(AddedRowsScanTask task) { - DataFile file = task.file(); - List deletes = task.deletes(); - return Stream.concat(Stream.of(file), deletes.stream()); + return InputFilesDecryptor.fromChangelogTasks(taskGroup().tasks(), encryptingFileIO()); } } diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/PositionDeletesRowReader.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/PositionDeletesRowReader.java index b11a16dcf079..9459bf23f12e 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/PositionDeletesRowReader.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/PositionDeletesRowReader.java @@ -19,7 +19,6 @@ package org.apache.iceberg.spark.source; import java.util.Map; -import org.apache.iceberg.ContentFile; import org.apache.iceberg.PositionDeletesScanTask; import org.apache.iceberg.ScanTaskGroup; import org.apache.iceberg.Schema; @@ -67,12 +66,7 @@ class PositionDeletesRowReader extends BaseRowReader @Override protected InputFilesDecryptor newInputFilesDecryptor() { - return InputFilesDecryptor.fromFiles( - () -> - taskGroup().tasks().stream() - .>map(PositionDeletesScanTask::file) - .iterator(), - encryptingFileIO()); + return InputFilesDecryptor.fromPositionDeletesTasks(taskGroup().tasks(), encryptingFileIO()); } @SuppressWarnings("resource") // handled by BaseReader From 14b252dd6a57fbd8c216b0b6e3fd59bb7c96b5b5 Mon Sep 17 00:00:00 2001 From: Gabor Kaszab Date: Wed, 4 Nov 2026 11:06:15 +0100 Subject: [PATCH 3/9] Add test coverage for changeog and position delete tasks --- .../encryption/InputFilesDecryptor.java | 3 - .../encryption/TestInputFilesDecryptor.java | 142 +++++++++++++++++- 2 files changed, 135 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java b/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java index 6c0b40ddab63..a3918491b9f3 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java +++ b/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java @@ -27,7 +27,6 @@ import org.apache.iceberg.ContentFile; import org.apache.iceberg.DeleteFile; import org.apache.iceberg.DeletedDataFileScanTask; -import org.apache.iceberg.DeletedRowsScanTask; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.PositionDeletesScanTask; import org.apache.iceberg.io.FileIO; @@ -112,8 +111,6 @@ private static Collection> changelogReferencedFiles( Stream.of(deletedDataFileTask.file()), deletedDataFileTask.existingDeletes().stream()) .forEach(file -> files.put(file.location(), file)); - } else if (task instanceof DeletedRowsScanTask) { - throw new UnsupportedOperationException("Deleted rows scan task is not supported yet"); } else { throw new IllegalArgumentException( "Unsupported changelog scan task type: " + task.getClass().getName()); diff --git a/core/src/test/java/org/apache/iceberg/encryption/TestInputFilesDecryptor.java b/core/src/test/java/org/apache/iceberg/encryption/TestInputFilesDecryptor.java index a0b82ba61e3f..f62fc73fbb42 100644 --- a/core/src/test/java/org/apache/iceberg/encryption/TestInputFilesDecryptor.java +++ b/core/src/test/java/org/apache/iceberg/encryption/TestInputFilesDecryptor.java @@ -24,18 +24,23 @@ import static org.mockito.ArgumentMatchers.any; import java.util.List; +import java.util.Map; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import org.apache.iceberg.AddedRowsScanTask; +import org.apache.iceberg.ChangelogScanTask; import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.DeletedDataFileScanTask; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileMetadata; import org.apache.iceberg.MockFileScanTask; import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.PositionDeletesScanTask; import org.apache.iceberg.io.InputFile; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; @@ -74,20 +79,48 @@ class TestInputFilesDecryptor { .withFormat(FileFormat.PARQUET) .build(); + private static final DeleteFile OTHER_DELETE_FILE = + FileMetadata.deleteFileBuilder(SPEC) + .ofPositionDeletes() + .withPath("/path/to/other-deletes.parquet") + .withFileSizeInBytes(54L) + .withRecordCount(2L) + .withFormat(FileFormat.PARQUET) + .build(); + private EncryptingFileIO encryptingIO; private InputFile dataInputFile; private InputFile deleteInputFile; + private InputFile otherDataInputFile; + private InputFile otherDeleteInputFile; + private Map inputFiles; @BeforeEach void before() { this.dataInputFile = Mockito.mock(InputFile.class); this.deleteInputFile = Mockito.mock(InputFile.class); + this.otherDataInputFile = Mockito.mock(InputFile.class); + this.otherDeleteInputFile = Mockito.mock(InputFile.class); + this.inputFiles = + ImmutableMap.of( + DATA_FILE.location(), + dataInputFile, + DELETE_FILE.location(), + deleteInputFile, + OTHER_DATA_FILE.location(), + otherDataInputFile, + OTHER_DELETE_FILE.location(), + otherDeleteInputFile); this.encryptingIO = Mockito.mock(EncryptingFileIO.class); Mockito.when(encryptingIO.bulkDecrypt(any())) - .thenReturn( - ImmutableMap.of( - DATA_FILE.location(), dataInputFile, - DELETE_FILE.location(), deleteInputFile)); + .thenAnswer( + invocation -> { + Iterable> files = invocation.getArgument(0); + ImmutableMap.Builder decrypted = ImmutableMap.builder(); + files.forEach( + file -> decrypted.put(file.location(), inputFiles.get(file.location()))); + return decrypted.buildOrThrow(); + }); } @Test @@ -143,12 +176,13 @@ void deduplicatesFiles() { @Test void resolvesInputFilesOnceWhenAccessedConcurrently() throws Exception { - Mockito.when(encryptingIO.bulkDecrypt(any())) - .thenAnswer( + Mockito.doAnswer( invocation -> { Thread.sleep(50); return ImmutableMap.of(DATA_FILE.location(), dataInputFile); - }); + }) + .when(encryptingIO) + .bulkDecrypt(any()); InputFilesDecryptor decryptor = InputFilesDecryptor.fromTasks( @@ -177,4 +211,98 @@ void resolvesInputFilesOnceWhenAccessedConcurrently() throws Exception { Mockito.verify(encryptingIO, Mockito.times(1)).bulkDecrypt(any()); } + + @Test + void resolvesFilesOfAddedRowsTasks() { + InputFilesDecryptor decryptor = + InputFilesDecryptor.fromChangelogTasks( + ImmutableList.of(addedRowsTask(DATA_FILE, DELETE_FILE)), encryptingIO); + + assertThat(decryptor.getInputFile(DATA_FILE.location())).isSameAs(dataInputFile); + assertThat(decryptor.getInputFile(DELETE_FILE.location())).isSameAs(deleteInputFile); + + Mockito.verify(encryptingIO, Mockito.times(1)).bulkDecrypt(any()); + } + + @Test + void resolvesFilesOfDeletedDataFileTasks() { + InputFilesDecryptor decryptor = + InputFilesDecryptor.fromChangelogTasks( + ImmutableList.of(deletedDataFileTask(DATA_FILE, DELETE_FILE)), encryptingIO); + + assertThat(decryptor.getInputFile(DATA_FILE.location())).isSameAs(dataInputFile); + assertThat(decryptor.getInputFile(DELETE_FILE.location())).isSameAs(deleteInputFile); + + Mockito.verify(encryptingIO, Mockito.times(1)).bulkDecrypt(any()); + } + + @Test + void deduplicatesFilesOfChangelogTasks() { + InputFilesDecryptor decryptor = + InputFilesDecryptor.fromChangelogTasks( + ImmutableList.of( + addedRowsTask(DATA_FILE, DELETE_FILE), + deletedDataFileTask(OTHER_DATA_FILE, DELETE_FILE)), + encryptingIO); + + assertThat(decryptor.getInputFile(DATA_FILE.location())).isSameAs(dataInputFile); + assertThat(decryptor.getInputFile(OTHER_DATA_FILE.location())).isSameAs(otherDataInputFile); + assertThat(decryptor.getInputFile(DELETE_FILE.location())).isSameAs(deleteInputFile); + + @SuppressWarnings("unchecked") + ArgumentCaptor>> captor = ArgumentCaptor.forClass(Iterable.class); + Mockito.verify(encryptingIO).bulkDecrypt(captor.capture()); + assertThat(captor.getValue()) + .containsExactlyInAnyOrder(DATA_FILE, OTHER_DATA_FILE, DELETE_FILE); + } + + @Test + void failsForUnsupportedChangelogTasks() { + ChangelogScanTask task = Mockito.mock(ChangelogScanTask.class); + InputFilesDecryptor decryptor = + InputFilesDecryptor.fromChangelogTasks(ImmutableList.of(task), encryptingIO); + + assertThatThrownBy(() -> decryptor.getInputFile(DATA_FILE.location())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unsupported changelog scan task type: " + task.getClass().getName()); + } + + @Test + void deduplicatesFilesOfPositionDeletesTasks() { + InputFilesDecryptor decryptor = + InputFilesDecryptor.fromPositionDeletesTasks( + ImmutableList.of( + positionDeletesTask(DELETE_FILE), + positionDeletesTask(DELETE_FILE), + positionDeletesTask(OTHER_DELETE_FILE)), + encryptingIO); + + assertThat(decryptor.getInputFile(DELETE_FILE.location())).isSameAs(deleteInputFile); + assertThat(decryptor.getInputFile(OTHER_DELETE_FILE.location())).isSameAs(otherDeleteInputFile); + + @SuppressWarnings("unchecked") + ArgumentCaptor>> captor = ArgumentCaptor.forClass(Iterable.class); + Mockito.verify(encryptingIO).bulkDecrypt(captor.capture()); + assertThat(captor.getValue()).containsExactlyInAnyOrder(DELETE_FILE, OTHER_DELETE_FILE); + } + + private static ChangelogScanTask addedRowsTask(DataFile file, DeleteFile... deletes) { + AddedRowsScanTask task = Mockito.mock(AddedRowsScanTask.class); + Mockito.when(task.file()).thenReturn(file); + Mockito.when(task.deletes()).thenReturn(ImmutableList.copyOf(deletes)); + return task; + } + + private static ChangelogScanTask deletedDataFileTask(DataFile file, DeleteFile... deletes) { + DeletedDataFileScanTask task = Mockito.mock(DeletedDataFileScanTask.class); + Mockito.when(task.file()).thenReturn(file); + Mockito.when(task.existingDeletes()).thenReturn(ImmutableList.copyOf(deletes)); + return task; + } + + private static PositionDeletesScanTask positionDeletesTask(DeleteFile file) { + PositionDeletesScanTask task = Mockito.mock(PositionDeletesScanTask.class); + Mockito.when(task.file()).thenReturn(file); + return task; + } } From d31e5105e2460e322b721e3bb0ac7a8fbec2666c Mon Sep 17 00:00:00 2001 From: Gabor Kaszab Date: Wed, 4 Nov 2026 11:36:50 +0100 Subject: [PATCH 4/9] Introduce more generic ScanTask based builder function --- .../encryption/InputFilesDecryptor.java | 62 ++++-------- .../encryption/TestInputFilesDecryptor.java | 98 +++++-------------- .../iceberg/spark/source/BaseReader.java | 18 +--- .../iceberg/spark/source/BatchDataReader.java | 6 -- .../spark/source/ChangelogRowReader.java | 6 -- .../source/PositionDeletesRowReader.java | 6 -- .../iceberg/spark/source/RowDataReader.java | 6 -- .../iceberg/spark/source/TestBaseReader.java | 6 -- 8 files changed, 43 insertions(+), 165 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java b/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java index a3918491b9f3..69d1487e36c8 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java +++ b/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java @@ -22,13 +22,12 @@ import java.util.Map; import java.util.stream.Stream; import org.apache.iceberg.AddedRowsScanTask; -import org.apache.iceberg.ChangelogScanTask; import org.apache.iceberg.CombinedScanTask; import org.apache.iceberg.ContentFile; -import org.apache.iceberg.DeleteFile; import org.apache.iceberg.DeletedDataFileScanTask; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.PositionDeletesScanTask; +import org.apache.iceberg.ScanTask; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; @@ -54,21 +53,10 @@ public InputFilesDecryptor( } public static InputFilesDecryptor fromTasks( - Iterable tasks, EncryptingFileIO encryptingIO) { + Iterable tasks, EncryptingFileIO encryptingIO) { return new InputFilesDecryptor(() -> referencedFiles(tasks).iterator(), encryptingIO); } - public static InputFilesDecryptor fromChangelogTasks( - Iterable tasks, EncryptingFileIO encryptingIO) { - return new InputFilesDecryptor(() -> changelogReferencedFiles(tasks).iterator(), encryptingIO); - } - - public static InputFilesDecryptor fromPositionDeletesTasks( - Iterable tasks, EncryptingFileIO encryptingIO) { - return new InputFilesDecryptor( - () -> positionDeletesReferencedFiles(tasks).iterator(), encryptingIO); - } - private InputFilesDecryptor( Iterable> files, EncryptingFileIO encryptingIO) { this.referencedFiles = files; @@ -87,44 +75,28 @@ private Map inputFiles() { return lazyInputFiles; } - private static Collection> referencedFiles(Iterable tasks) { - Map> files = Maps.newHashMap(); - for (FileScanTask task : tasks) { - files.put(task.file().location(), task.file()); - for (DeleteFile delete : task.deletes()) { - files.put(delete.location(), delete); - } - } - - return files.values(); - } - - private static Collection> changelogReferencedFiles( - Iterable tasks) { + private static Collection> referencedFiles(Iterable tasks) { Map> files = Maps.newHashMap(); - for (ChangelogScanTask task : tasks) { - if (task instanceof AddedRowsScanTask addedRowsTask) { - Stream.concat(Stream.of(addedRowsTask.file()), addedRowsTask.deletes().stream()) - .forEach(file -> files.put(file.location(), file)); + for (ScanTask task : tasks) { + Stream> taskFiles; + if (task instanceof FileScanTask fileTask) { + taskFiles = Stream.concat(Stream.of(fileTask.file()), fileTask.deletes().stream()); + } else if (task instanceof AddedRowsScanTask addedRowsTask) { + taskFiles = + Stream.concat(Stream.of(addedRowsTask.file()), addedRowsTask.deletes().stream()); } else if (task instanceof DeletedDataFileScanTask deletedDataFileTask) { - Stream.concat( + taskFiles = + Stream.concat( Stream.of(deletedDataFileTask.file()), - deletedDataFileTask.existingDeletes().stream()) - .forEach(file -> files.put(file.location(), file)); + deletedDataFileTask.existingDeletes().stream()); + } else if (task instanceof PositionDeletesScanTask positionDeletesTask) { + taskFiles = Stream.of(positionDeletesTask.file()); } else { throw new IllegalArgumentException( - "Unsupported changelog scan task type: " + task.getClass().getName()); + "Unsupported scan task type: " + task.getClass().getName()); } - } - return files.values(); - } - - private static Collection> positionDeletesReferencedFiles( - Iterable tasks) { - Map> files = Maps.newHashMap(); - for (PositionDeletesScanTask task : tasks) { - files.put(task.file().location(), task.file()); + taskFiles.forEach(file -> files.put(file.location(), file)); } return files.values(); diff --git a/core/src/test/java/org/apache/iceberg/encryption/TestInputFilesDecryptor.java b/core/src/test/java/org/apache/iceberg/encryption/TestInputFilesDecryptor.java index f62fc73fbb42..fa0f29426435 100644 --- a/core/src/test/java/org/apache/iceberg/encryption/TestInputFilesDecryptor.java +++ b/core/src/test/java/org/apache/iceberg/encryption/TestInputFilesDecryptor.java @@ -30,7 +30,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import org.apache.iceberg.AddedRowsScanTask; -import org.apache.iceberg.ChangelogScanTask; import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; @@ -41,6 +40,7 @@ import org.apache.iceberg.MockFileScanTask; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PositionDeletesScanTask; +import org.apache.iceberg.ScanTask; import org.apache.iceberg.io.InputFile; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; @@ -79,20 +79,10 @@ class TestInputFilesDecryptor { .withFormat(FileFormat.PARQUET) .build(); - private static final DeleteFile OTHER_DELETE_FILE = - FileMetadata.deleteFileBuilder(SPEC) - .ofPositionDeletes() - .withPath("/path/to/other-deletes.parquet") - .withFileSizeInBytes(54L) - .withRecordCount(2L) - .withFormat(FileFormat.PARQUET) - .build(); - private EncryptingFileIO encryptingIO; private InputFile dataInputFile; private InputFile deleteInputFile; private InputFile otherDataInputFile; - private InputFile otherDeleteInputFile; private Map inputFiles; @BeforeEach @@ -100,7 +90,6 @@ void before() { this.dataInputFile = Mockito.mock(InputFile.class); this.deleteInputFile = Mockito.mock(InputFile.class); this.otherDataInputFile = Mockito.mock(InputFile.class); - this.otherDeleteInputFile = Mockito.mock(InputFile.class); this.inputFiles = ImmutableMap.of( DATA_FILE.location(), @@ -108,9 +97,7 @@ void before() { DELETE_FILE.location(), deleteInputFile, OTHER_DATA_FILE.location(), - otherDataInputFile, - OTHER_DELETE_FILE.location(), - otherDeleteInputFile); + otherDataInputFile); this.encryptingIO = Mockito.mock(EncryptingFileIO.class); Mockito.when(encryptingIO.bulkDecrypt(any())) .thenAnswer( @@ -214,9 +201,12 @@ void resolvesInputFilesOnceWhenAccessedConcurrently() throws Exception { @Test void resolvesFilesOfAddedRowsTasks() { + AddedRowsScanTask task = Mockito.mock(AddedRowsScanTask.class); + Mockito.when(task.file()).thenReturn(DATA_FILE); + Mockito.when(task.deletes()).thenReturn(ImmutableList.of(DELETE_FILE)); + InputFilesDecryptor decryptor = - InputFilesDecryptor.fromChangelogTasks( - ImmutableList.of(addedRowsTask(DATA_FILE, DELETE_FILE)), encryptingIO); + InputFilesDecryptor.fromTasks(ImmutableList.of(task), encryptingIO); assertThat(decryptor.getInputFile(DATA_FILE.location())).isSameAs(dataInputFile); assertThat(decryptor.getInputFile(DELETE_FILE.location())).isSameAs(deleteInputFile); @@ -226,9 +216,12 @@ void resolvesFilesOfAddedRowsTasks() { @Test void resolvesFilesOfDeletedDataFileTasks() { + DeletedDataFileScanTask task = Mockito.mock(DeletedDataFileScanTask.class); + Mockito.when(task.file()).thenReturn(DATA_FILE); + Mockito.when(task.existingDeletes()).thenReturn(ImmutableList.of(DELETE_FILE)); + InputFilesDecryptor decryptor = - InputFilesDecryptor.fromChangelogTasks( - ImmutableList.of(deletedDataFileTask(DATA_FILE, DELETE_FILE)), encryptingIO); + InputFilesDecryptor.fromTasks(ImmutableList.of(task), encryptingIO); assertThat(decryptor.getInputFile(DATA_FILE.location())).isSameAs(dataInputFile); assertThat(decryptor.getInputFile(DELETE_FILE.location())).isSameAs(deleteInputFile); @@ -237,72 +230,27 @@ void resolvesFilesOfDeletedDataFileTasks() { } @Test - void deduplicatesFilesOfChangelogTasks() { + void resolvesFilesOfPositionDeletesTasks() { + PositionDeletesScanTask task = Mockito.mock(PositionDeletesScanTask.class); + Mockito.when(task.file()).thenReturn(DELETE_FILE); + InputFilesDecryptor decryptor = - InputFilesDecryptor.fromChangelogTasks( - ImmutableList.of( - addedRowsTask(DATA_FILE, DELETE_FILE), - deletedDataFileTask(OTHER_DATA_FILE, DELETE_FILE)), - encryptingIO); + InputFilesDecryptor.fromTasks(ImmutableList.of(task), encryptingIO); - assertThat(decryptor.getInputFile(DATA_FILE.location())).isSameAs(dataInputFile); - assertThat(decryptor.getInputFile(OTHER_DATA_FILE.location())).isSameAs(otherDataInputFile); assertThat(decryptor.getInputFile(DELETE_FILE.location())).isSameAs(deleteInputFile); - @SuppressWarnings("unchecked") - ArgumentCaptor>> captor = ArgumentCaptor.forClass(Iterable.class); - Mockito.verify(encryptingIO).bulkDecrypt(captor.capture()); - assertThat(captor.getValue()) - .containsExactlyInAnyOrder(DATA_FILE, OTHER_DATA_FILE, DELETE_FILE); + Mockito.verify(encryptingIO, Mockito.times(1)).bulkDecrypt(any()); } @Test - void failsForUnsupportedChangelogTasks() { - ChangelogScanTask task = Mockito.mock(ChangelogScanTask.class); + void failsForUnsupportedTasks() { + ScanTask task = Mockito.mock(ScanTask.class); + InputFilesDecryptor decryptor = - InputFilesDecryptor.fromChangelogTasks(ImmutableList.of(task), encryptingIO); + InputFilesDecryptor.fromTasks(ImmutableList.of(task), encryptingIO); assertThatThrownBy(() -> decryptor.getInputFile(DATA_FILE.location())) .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Unsupported changelog scan task type: " + task.getClass().getName()); - } - - @Test - void deduplicatesFilesOfPositionDeletesTasks() { - InputFilesDecryptor decryptor = - InputFilesDecryptor.fromPositionDeletesTasks( - ImmutableList.of( - positionDeletesTask(DELETE_FILE), - positionDeletesTask(DELETE_FILE), - positionDeletesTask(OTHER_DELETE_FILE)), - encryptingIO); - - assertThat(decryptor.getInputFile(DELETE_FILE.location())).isSameAs(deleteInputFile); - assertThat(decryptor.getInputFile(OTHER_DELETE_FILE.location())).isSameAs(otherDeleteInputFile); - - @SuppressWarnings("unchecked") - ArgumentCaptor>> captor = ArgumentCaptor.forClass(Iterable.class); - Mockito.verify(encryptingIO).bulkDecrypt(captor.capture()); - assertThat(captor.getValue()).containsExactlyInAnyOrder(DELETE_FILE, OTHER_DELETE_FILE); - } - - private static ChangelogScanTask addedRowsTask(DataFile file, DeleteFile... deletes) { - AddedRowsScanTask task = Mockito.mock(AddedRowsScanTask.class); - Mockito.when(task.file()).thenReturn(file); - Mockito.when(task.deletes()).thenReturn(ImmutableList.copyOf(deletes)); - return task; - } - - private static ChangelogScanTask deletedDataFileTask(DataFile file, DeleteFile... deletes) { - DeletedDataFileScanTask task = Mockito.mock(DeletedDataFileScanTask.class); - Mockito.when(task.file()).thenReturn(file); - Mockito.when(task.existingDeletes()).thenReturn(ImmutableList.copyOf(deletes)); - return task; - } - - private static PositionDeletesScanTask positionDeletesTask(DeleteFile file) { - PositionDeletesScanTask task = Mockito.mock(PositionDeletesScanTask.class); - Mockito.when(task.file()).thenReturn(file); - return task; + .hasMessage("Unsupported scan task type: " + task.getClass().getName()); } } diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java index ca43a81523d0..5e1fa357bce9 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java @@ -68,11 +68,9 @@ abstract class BaseReader implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(BaseReader.class); private final Table table; - private final EncryptingFileIO fileIO; private final Schema expectedSchema; private final boolean caseSensitive; private final NameMapping nameMapping; - private final ScanTaskGroup taskGroup; private final Iterator tasks; private final DeleteCounter counter; private final boolean cacheDeleteFilesOnExecutors; @@ -90,8 +88,6 @@ abstract class BaseReader implements Closeable { boolean caseSensitive, boolean cacheDeleteFilesOnExecutors) { this.table = table; - this.fileIO = EncryptingFileIO.combine(fileIO, table().encryption()); - this.taskGroup = taskGroup; this.tasks = taskGroup.tasks().iterator(); this.currentIterator = CloseableIterator.empty(); this.expectedSchema = expectedSchema; @@ -101,13 +97,13 @@ abstract class BaseReader implements Closeable { nameMappingString != null ? NameMappingParser.fromJson(nameMappingString) : null; this.counter = new DeleteCounter(); this.cacheDeleteFilesOnExecutors = cacheDeleteFilesOnExecutors; - this.inputFilesDecryptor = newInputFilesDecryptor(); + this.inputFilesDecryptor = + InputFilesDecryptor.fromTasks( + taskGroup.tasks(), EncryptingFileIO.combine(fileIO, table.encryption())); } protected abstract CloseableIterator open(TaskT task); - protected abstract InputFilesDecryptor newInputFilesDecryptor(); - protected Schema expectedSchema() { return expectedSchema; } @@ -176,14 +172,6 @@ protected InputFile getInputFile(String location) { return inputFilesDecryptor.getInputFile(location); } - protected ScanTaskGroup taskGroup() { - return taskGroup; - } - - protected EncryptingFileIO encryptingFileIO() { - return fileIO; - } - protected Map constantsMap(ContentScanTask task, Schema readSchema) { if (readSchema.findField(MetadataColumns.PARTITION_COLUMN_ID) != null) { StructType partitionType = Partitioning.partitionType(table); diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BatchDataReader.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BatchDataReader.java index 7c7a3242fd2c..327ad93753c2 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BatchDataReader.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BatchDataReader.java @@ -23,7 +23,6 @@ import org.apache.iceberg.ScanTaskGroup; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; -import org.apache.iceberg.encryption.InputFilesDecryptor; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; @@ -90,11 +89,6 @@ public CustomTaskMetric[] currentMetricsValues() { }; } - @Override - protected InputFilesDecryptor newInputFilesDecryptor() { - return InputFilesDecryptor.fromTasks(taskGroup().tasks(), encryptingFileIO()); - } - @Override protected CloseableIterator open(FileScanTask task) { String filePath = task.file().location(); diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/ChangelogRowReader.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/ChangelogRowReader.java index 03e57d6d834f..2343ae774768 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/ChangelogRowReader.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/ChangelogRowReader.java @@ -29,7 +29,6 @@ import org.apache.iceberg.ScanTaskGroup; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; -import org.apache.iceberg.encryption.InputFilesDecryptor; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.FileIO; @@ -139,9 +138,4 @@ private CloseableIterable rows(ContentScanTask task, Sche readSchema, idToConstant); } - - @Override - protected InputFilesDecryptor newInputFilesDecryptor() { - return InputFilesDecryptor.fromChangelogTasks(taskGroup().tasks(), encryptingFileIO()); - } } diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/PositionDeletesRowReader.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/PositionDeletesRowReader.java index 9459bf23f12e..06b702fef753 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/PositionDeletesRowReader.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/PositionDeletesRowReader.java @@ -23,7 +23,6 @@ import org.apache.iceberg.ScanTaskGroup; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; -import org.apache.iceberg.encryption.InputFilesDecryptor; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.ExpressionUtil; import org.apache.iceberg.io.CloseableIterator; @@ -64,11 +63,6 @@ class PositionDeletesRowReader extends BaseRowReader LOG.debug("Reading {} position delete file split(s) for table {}", numSplits, table.name()); } - @Override - protected InputFilesDecryptor newInputFilesDecryptor() { - return InputFilesDecryptor.fromPositionDeletesTasks(taskGroup().tasks(), encryptingFileIO()); - } - @SuppressWarnings("resource") // handled by BaseReader @Override protected CloseableIterator open(PositionDeletesScanTask task) { diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/RowDataReader.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/RowDataReader.java index aacbe282f2c7..25f60e93154a 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/RowDataReader.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/RowDataReader.java @@ -24,7 +24,6 @@ import org.apache.iceberg.ScanTaskGroup; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; -import org.apache.iceberg.encryption.InputFilesDecryptor; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.FileIO; @@ -74,11 +73,6 @@ public CustomTaskMetric[] currentMetricsValues() { }; } - @Override - protected InputFilesDecryptor newInputFilesDecryptor() { - return InputFilesDecryptor.fromTasks(taskGroup().tasks(), encryptingFileIO()); - } - @Override protected CloseableIterator open(FileScanTask task) { String filePath = task.file().location(); diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestBaseReader.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestBaseReader.java index f5d0fef3d70f..71bb111e54e8 100644 --- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestBaseReader.java +++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestBaseReader.java @@ -41,7 +41,6 @@ import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; -import org.apache.iceberg.encryption.InputFilesDecryptor; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.FileAppender; import org.apache.iceberg.parquet.Parquet; @@ -92,11 +91,6 @@ private static class ClosureTrackingReader extends BaseReader open(FileScanTask task) { CloseableIntegerRange intRange = new CloseableIntegerRange(task.file().recordCount()); From d0890afc12747089464a1c8968eef5d1f848e63a Mon Sep 17 00:00:00 2001 From: Gabor Kaszab Date: Wed, 4 Nov 2026 17:04:52 +0100 Subject: [PATCH 5/9] Change ScanTask implementation to return their referenced files --- .../org/apache/iceberg/AddedRowsScanTask.java | 10 +++- .../iceberg/DeletedDataFileScanTask.java | 10 +++- .../apache/iceberg/DeletedRowsScanTask.java | 10 +++- .../java/org/apache/iceberg/FileScanTask.java | 10 +++- .../apache/iceberg/HasReferencedFiles.java | 29 +++++++++++ .../iceberg/PositionDeletesScanTask.java | 9 +++- .../encryption/InputFilesDecryptor.java | 22 ++------ .../encryption/TestInputFilesDecryptor.java | 51 +++++++++++++++---- 8 files changed, 118 insertions(+), 33 deletions(-) create mode 100644 api/src/main/java/org/apache/iceberg/HasReferencedFiles.java diff --git a/api/src/main/java/org/apache/iceberg/AddedRowsScanTask.java b/api/src/main/java/org/apache/iceberg/AddedRowsScanTask.java index 506e344d3660..0d09276c8d97 100644 --- a/api/src/main/java/org/apache/iceberg/AddedRowsScanTask.java +++ b/api/src/main/java/org/apache/iceberg/AddedRowsScanTask.java @@ -19,6 +19,8 @@ package org.apache.iceberg; import java.util.List; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.util.ScanTaskUtil; /** @@ -41,7 +43,8 @@ *

Readers consuming these tasks should produce added records with metadata like change ordinal * and commit snapshot ID. */ -public interface AddedRowsScanTask extends ChangelogScanTask, ContentScanTask { +public interface AddedRowsScanTask + extends ChangelogScanTask, ContentScanTask, HasReferencedFiles { /** * A list of {@link DeleteFile delete files} to apply when reading the data file in this task. * @@ -63,4 +66,9 @@ default long sizeBytes() { default int filesCount() { return 1 + deletes().size(); } + + @Override + default Iterable> referencedFiles() { + return Iterables.concat(ImmutableList.of(file()), deletes()); + } } diff --git a/api/src/main/java/org/apache/iceberg/DeletedDataFileScanTask.java b/api/src/main/java/org/apache/iceberg/DeletedDataFileScanTask.java index 4b9c1704b9d2..abcf24f443ec 100644 --- a/api/src/main/java/org/apache/iceberg/DeletedDataFileScanTask.java +++ b/api/src/main/java/org/apache/iceberg/DeletedDataFileScanTask.java @@ -19,6 +19,8 @@ package org.apache.iceberg; import java.util.List; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.util.ScanTaskUtil; /** @@ -39,7 +41,8 @@ *

Readers consuming these tasks should produce deleted records with metadata like change ordinal * and commit snapshot ID. */ -public interface DeletedDataFileScanTask extends ChangelogScanTask, ContentScanTask { +public interface DeletedDataFileScanTask + extends ChangelogScanTask, ContentScanTask, HasReferencedFiles { /** * A list of previously added {@link DeleteFile delete files} to apply when reading the data file * in this task. @@ -62,4 +65,9 @@ default long sizeBytes() { default int filesCount() { return 1 + existingDeletes().size(); } + + @Override + default Iterable> referencedFiles() { + return Iterables.concat(ImmutableList.of(file()), existingDeletes()); + } } diff --git a/api/src/main/java/org/apache/iceberg/DeletedRowsScanTask.java b/api/src/main/java/org/apache/iceberg/DeletedRowsScanTask.java index 1e0a52a53241..61b6115e042c 100644 --- a/api/src/main/java/org/apache/iceberg/DeletedRowsScanTask.java +++ b/api/src/main/java/org/apache/iceberg/DeletedRowsScanTask.java @@ -19,6 +19,8 @@ package org.apache.iceberg; import java.util.List; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.util.ScanTaskUtil; /** @@ -39,7 +41,8 @@ *

Readers consuming these tasks should produce deleted records with metadata like change ordinal * and commit snapshot ID. */ -public interface DeletedRowsScanTask extends ChangelogScanTask, ContentScanTask { +public interface DeletedRowsScanTask + extends ChangelogScanTask, ContentScanTask, HasReferencedFiles { /** * A list of added {@link DeleteFile delete files} that apply to the task's data file. Records * removed by these delete files should appear as deletes in the changelog. @@ -73,4 +76,9 @@ default long sizeBytes() { default int filesCount() { return 1 + addedDeletes().size() + existingDeletes().size(); } + + @Override + default Iterable> referencedFiles() { + return Iterables.concat(ImmutableList.of(file()), addedDeletes(), existingDeletes()); + } } diff --git a/api/src/main/java/org/apache/iceberg/FileScanTask.java b/api/src/main/java/org/apache/iceberg/FileScanTask.java index 94f153e56052..0c1169d3ecb7 100644 --- a/api/src/main/java/org/apache/iceberg/FileScanTask.java +++ b/api/src/main/java/org/apache/iceberg/FileScanTask.java @@ -19,10 +19,13 @@ package org.apache.iceberg; import java.util.List; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.util.ScanTaskUtil; /** A scan task over a range of bytes in a single data file. */ -public interface FileScanTask extends ContentScanTask, SplittableScanTask { +public interface FileScanTask + extends ContentScanTask, SplittableScanTask, HasReferencedFiles { /** * A list of {@link DeleteFile delete files} to apply when reading the task's data file. * @@ -45,6 +48,11 @@ default int filesCount() { return 1 + deletes().size(); } + @Override + default Iterable> referencedFiles() { + return Iterables.concat(ImmutableList.of(file()), deletes()); + } + @Override default boolean isFileScanTask() { return true; diff --git a/api/src/main/java/org/apache/iceberg/HasReferencedFiles.java b/api/src/main/java/org/apache/iceberg/HasReferencedFiles.java new file mode 100644 index 000000000000..ef2388106a27 --- /dev/null +++ b/api/src/main/java/org/apache/iceberg/HasReferencedFiles.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +/** A {@link ScanTask} that can list the content files it reads. */ +public interface HasReferencedFiles { + /** + * Returns all content files that must be read to process this task + * + * @return the content files referenced by this task + */ + Iterable> referencedFiles(); +} diff --git a/core/src/main/java/org/apache/iceberg/PositionDeletesScanTask.java b/core/src/main/java/org/apache/iceberg/PositionDeletesScanTask.java index e8ae8a08cd68..b71ed43e5236 100644 --- a/core/src/main/java/org/apache/iceberg/PositionDeletesScanTask.java +++ b/core/src/main/java/org/apache/iceberg/PositionDeletesScanTask.java @@ -18,5 +18,12 @@ */ package org.apache.iceberg; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; + /** A {@link ScanTask} for position delete files */ -public interface PositionDeletesScanTask extends ContentScanTask {} +public interface PositionDeletesScanTask extends ContentScanTask, HasReferencedFiles { + @Override + default Iterable> referencedFiles() { + return ImmutableList.of(file()); + } +} diff --git a/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java b/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java index 69d1487e36c8..3e98a09ebef1 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java +++ b/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java @@ -20,13 +20,10 @@ import java.util.Collection; import java.util.Map; -import java.util.stream.Stream; -import org.apache.iceberg.AddedRowsScanTask; import org.apache.iceberg.CombinedScanTask; import org.apache.iceberg.ContentFile; -import org.apache.iceberg.DeletedDataFileScanTask; import org.apache.iceberg.FileScanTask; -import org.apache.iceberg.PositionDeletesScanTask; +import org.apache.iceberg.HasReferencedFiles; import org.apache.iceberg.ScanTask; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; @@ -78,25 +75,12 @@ private Map inputFiles() { private static Collection> referencedFiles(Iterable tasks) { Map> files = Maps.newHashMap(); for (ScanTask task : tasks) { - Stream> taskFiles; - if (task instanceof FileScanTask fileTask) { - taskFiles = Stream.concat(Stream.of(fileTask.file()), fileTask.deletes().stream()); - } else if (task instanceof AddedRowsScanTask addedRowsTask) { - taskFiles = - Stream.concat(Stream.of(addedRowsTask.file()), addedRowsTask.deletes().stream()); - } else if (task instanceof DeletedDataFileScanTask deletedDataFileTask) { - taskFiles = - Stream.concat( - Stream.of(deletedDataFileTask.file()), - deletedDataFileTask.existingDeletes().stream()); - } else if (task instanceof PositionDeletesScanTask positionDeletesTask) { - taskFiles = Stream.of(positionDeletesTask.file()); - } else { + if (!(task instanceof HasReferencedFiles referencing)) { throw new IllegalArgumentException( "Unsupported scan task type: " + task.getClass().getName()); } - taskFiles.forEach(file -> files.put(file.location(), file)); + referencing.referencedFiles().forEach(file -> files.put(file.location(), file)); } return files.values(); diff --git a/core/src/test/java/org/apache/iceberg/encryption/TestInputFilesDecryptor.java b/core/src/test/java/org/apache/iceberg/encryption/TestInputFilesDecryptor.java index fa0f29426435..0e91d77d081b 100644 --- a/core/src/test/java/org/apache/iceberg/encryption/TestInputFilesDecryptor.java +++ b/core/src/test/java/org/apache/iceberg/encryption/TestInputFilesDecryptor.java @@ -35,6 +35,7 @@ import org.apache.iceberg.DataFiles; import org.apache.iceberg.DeleteFile; import org.apache.iceberg.DeletedDataFileScanTask; +import org.apache.iceberg.DeletedRowsScanTask; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileMetadata; import org.apache.iceberg.MockFileScanTask; @@ -79,10 +80,20 @@ class TestInputFilesDecryptor { .withFormat(FileFormat.PARQUET) .build(); + private static final DeleteFile OTHER_DELETE_FILE = + FileMetadata.deleteFileBuilder(SPEC) + .ofPositionDeletes() + .withPath("/path/to/other-deletes.parquet") + .withFileSizeInBytes(54L) + .withRecordCount(2L) + .withFormat(FileFormat.PARQUET) + .build(); + private EncryptingFileIO encryptingIO; private InputFile dataInputFile; private InputFile deleteInputFile; private InputFile otherDataInputFile; + private InputFile otherDeleteInputFile; private Map inputFiles; @BeforeEach @@ -90,6 +101,7 @@ void before() { this.dataInputFile = Mockito.mock(InputFile.class); this.deleteInputFile = Mockito.mock(InputFile.class); this.otherDataInputFile = Mockito.mock(InputFile.class); + this.otherDeleteInputFile = Mockito.mock(InputFile.class); this.inputFiles = ImmutableMap.of( DATA_FILE.location(), @@ -97,7 +109,9 @@ void before() { DELETE_FILE.location(), deleteInputFile, OTHER_DATA_FILE.location(), - otherDataInputFile); + otherDataInputFile, + OTHER_DELETE_FILE.location(), + otherDeleteInputFile); this.encryptingIO = Mockito.mock(EncryptingFileIO.class); Mockito.when(encryptingIO.bulkDecrypt(any())) .thenAnswer( @@ -201,9 +215,9 @@ void resolvesInputFilesOnceWhenAccessedConcurrently() throws Exception { @Test void resolvesFilesOfAddedRowsTasks() { - AddedRowsScanTask task = Mockito.mock(AddedRowsScanTask.class); - Mockito.when(task.file()).thenReturn(DATA_FILE); - Mockito.when(task.deletes()).thenReturn(ImmutableList.of(DELETE_FILE)); + AddedRowsScanTask task = Mockito.mock(AddedRowsScanTask.class, Mockito.CALLS_REAL_METHODS); + Mockito.doReturn(DATA_FILE).when(task).file(); + Mockito.doReturn(ImmutableList.of(DELETE_FILE)).when(task).deletes(); InputFilesDecryptor decryptor = InputFilesDecryptor.fromTasks(ImmutableList.of(task), encryptingIO); @@ -216,23 +230,42 @@ void resolvesFilesOfAddedRowsTasks() { @Test void resolvesFilesOfDeletedDataFileTasks() { - DeletedDataFileScanTask task = Mockito.mock(DeletedDataFileScanTask.class); - Mockito.when(task.file()).thenReturn(DATA_FILE); - Mockito.when(task.existingDeletes()).thenReturn(ImmutableList.of(DELETE_FILE)); + DeletedDataFileScanTask task = + Mockito.mock(DeletedDataFileScanTask.class, Mockito.CALLS_REAL_METHODS); + Mockito.doReturn(DATA_FILE).when(task).file(); + Mockito.doReturn(ImmutableList.of(DELETE_FILE)).when(task).existingDeletes(); + + InputFilesDecryptor decryptor = + InputFilesDecryptor.fromTasks(ImmutableList.of(task), encryptingIO); + + assertThat(decryptor.getInputFile(DATA_FILE.location())).isSameAs(dataInputFile); + assertThat(decryptor.getInputFile(DELETE_FILE.location())).isSameAs(deleteInputFile); + + Mockito.verify(encryptingIO, Mockito.times(1)).bulkDecrypt(any()); + } + + @Test + void resolvesFilesOfDeletedRowsTasks() { + DeletedRowsScanTask task = Mockito.mock(DeletedRowsScanTask.class, Mockito.CALLS_REAL_METHODS); + Mockito.doReturn(DATA_FILE).when(task).file(); + Mockito.doReturn(ImmutableList.of(DELETE_FILE)).when(task).addedDeletes(); + Mockito.doReturn(ImmutableList.of(OTHER_DELETE_FILE)).when(task).existingDeletes(); InputFilesDecryptor decryptor = InputFilesDecryptor.fromTasks(ImmutableList.of(task), encryptingIO); assertThat(decryptor.getInputFile(DATA_FILE.location())).isSameAs(dataInputFile); assertThat(decryptor.getInputFile(DELETE_FILE.location())).isSameAs(deleteInputFile); + assertThat(decryptor.getInputFile(OTHER_DELETE_FILE.location())).isSameAs(otherDeleteInputFile); Mockito.verify(encryptingIO, Mockito.times(1)).bulkDecrypt(any()); } @Test void resolvesFilesOfPositionDeletesTasks() { - PositionDeletesScanTask task = Mockito.mock(PositionDeletesScanTask.class); - Mockito.when(task.file()).thenReturn(DELETE_FILE); + PositionDeletesScanTask task = + Mockito.mock(PositionDeletesScanTask.class, Mockito.CALLS_REAL_METHODS); + Mockito.doReturn(DELETE_FILE).when(task).file(); InputFilesDecryptor decryptor = InputFilesDecryptor.fromTasks(ImmutableList.of(task), encryptingIO); From 840fab82378f7e604108f6c609a3e78d160f56e9 Mon Sep 17 00:00:00 2001 From: Gabor Kaszab Date: Thu, 5 Nov 2026 09:41:47 +0100 Subject: [PATCH 6/9] Add javadoc and adjust logging --- .../apache/iceberg/encryption/InputFilesDecryptor.java | 6 ++++++ .../org/apache/iceberg/spark/source/BaseReader.java | 10 ++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java b/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java index 3e98a09ebef1..25e11d302a99 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java +++ b/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java @@ -49,6 +49,12 @@ public InputFilesDecryptor( EncryptingFileIO.combine(io, encryption)); } + /** + * Returns a decryptor for the file referenced by the given tasks. + * + *

Resolution of referenced files are performed lazily on the first call of {@link + * #getInputFile(String)}. + */ public static InputFilesDecryptor fromTasks( Iterable tasks, EncryptingFileIO encryptingIO) { return new InputFilesDecryptor(() -> referencedFiles(tasks).iterator(), encryptingIO); diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java index 5e1fa357bce9..8367315caaa4 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java @@ -27,8 +27,10 @@ import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; +import org.apache.iceberg.ContentFile; import org.apache.iceberg.ContentScanTask; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.HasReferencedFiles; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.Partitioning; import org.apache.iceberg.ScanTask; @@ -48,6 +50,7 @@ import org.apache.iceberg.io.InputFile; import org.apache.iceberg.mapping.NameMapping; import org.apache.iceberg.mapping.NameMappingParser; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.spark.SparkExecutorCache; import org.apache.iceberg.spark.SparkSchemaUtil; import org.apache.iceberg.spark.SparkUtil; @@ -144,8 +147,11 @@ public boolean next() throws IOException { } } } catch (IOException | RuntimeException e) { - if (currentTask instanceof ContentScanTask contentTask && !currentTask.isDataTask()) { - LOG.error("Error reading file: {}", contentTask.file().location(), e); + if (currentTask instanceof HasReferencedFiles referencing && !currentTask.isDataTask()) { + String filePaths = + String.join( + ", ", Iterables.transform(referencing.referencedFiles(), ContentFile::location)); + LOG.error("Error reading file(s): {}", filePaths, e); } throw e; } From a292ef899335fe7964c3499b41ce62883f29684e Mon Sep 17 00:00:00 2001 From: Gabor Kaszab Date: Thu, 5 Nov 2026 16:40:01 +0100 Subject: [PATCH 7/9] Precondition instead of instanceof, HasReferencedFiles comments --- .../java/org/apache/iceberg/HasReferencedFiles.java | 6 +++--- .../iceberg/encryption/InputFilesDecryptor.java | 13 +++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/HasReferencedFiles.java b/api/src/main/java/org/apache/iceberg/HasReferencedFiles.java index ef2388106a27..685fc23a772e 100644 --- a/api/src/main/java/org/apache/iceberg/HasReferencedFiles.java +++ b/api/src/main/java/org/apache/iceberg/HasReferencedFiles.java @@ -18,12 +18,12 @@ */ package org.apache.iceberg; -/** A {@link ScanTask} that can list the content files it reads. */ +/** An object that references content files. */ public interface HasReferencedFiles { /** - * Returns all content files that must be read to process this task + * Returns the content files referenced by this object. * - * @return the content files referenced by this task + * @return the referenced content files */ Iterable> referencedFiles(); } diff --git a/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java b/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java index 25e11d302a99..29dfff0101d1 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java +++ b/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java @@ -81,12 +81,13 @@ private Map inputFiles() { private static Collection> referencedFiles(Iterable tasks) { Map> files = Maps.newHashMap(); for (ScanTask task : tasks) { - if (!(task instanceof HasReferencedFiles referencing)) { - throw new IllegalArgumentException( - "Unsupported scan task type: " + task.getClass().getName()); - } - - referencing.referencedFiles().forEach(file -> files.put(file.location(), file)); + Preconditions.checkArgument( + task instanceof HasReferencedFiles, + "Unsupported scan task type: %s", + task.getClass().getName()); + ((HasReferencedFiles) task) + .referencedFiles() + .forEach(file -> files.put(file.location(), file)); } return files.values(); From e3518f8bd27daf9d1cb3756c63e92d4c12423bdc Mon Sep 17 00:00:00 2001 From: Gabor Kaszab Date: Thu, 5 Nov 2026 16:46:50 +0100 Subject: [PATCH 8/9] Fix typo in fromTasks() comment --- .../java/org/apache/iceberg/encryption/InputFilesDecryptor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java b/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java index 29dfff0101d1..75ba8c8f84b4 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java +++ b/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java @@ -50,7 +50,7 @@ public InputFilesDecryptor( } /** - * Returns a decryptor for the file referenced by the given tasks. + * Returns a decryptor for the files referenced by the given tasks. * *

Resolution of referenced files are performed lazily on the first call of {@link * #getInputFile(String)}. From 3626199d659e8a00910d0fb6f281bbfa7df1ef26 Mon Sep 17 00:00:00 2001 From: Gabor Kaszab Date: Wed, 11 Nov 2026 09:58:42 +0100 Subject: [PATCH 9/9] Remove HasReferencedFiles and move referencedFiles() into util --- .../org/apache/iceberg/AddedRowsScanTask.java | 10 +----- .../iceberg/DeletedDataFileScanTask.java | 10 +----- .../apache/iceberg/DeletedRowsScanTask.java | 10 +----- .../java/org/apache/iceberg/FileScanTask.java | 10 +----- .../apache/iceberg/HasReferencedFiles.java | 29 ---------------- .../iceberg/PositionDeletesScanTask.java | 9 +---- .../encryption/InputFilesDecryptor.java | 12 +++---- .../apache/iceberg/util/TableScanUtil.java | 34 +++++++++++++++++++ .../iceberg/spark/source/BaseReader.java | 10 +++--- 9 files changed, 48 insertions(+), 86 deletions(-) delete mode 100644 api/src/main/java/org/apache/iceberg/HasReferencedFiles.java diff --git a/api/src/main/java/org/apache/iceberg/AddedRowsScanTask.java b/api/src/main/java/org/apache/iceberg/AddedRowsScanTask.java index 0d09276c8d97..506e344d3660 100644 --- a/api/src/main/java/org/apache/iceberg/AddedRowsScanTask.java +++ b/api/src/main/java/org/apache/iceberg/AddedRowsScanTask.java @@ -19,8 +19,6 @@ package org.apache.iceberg; import java.util.List; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; -import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.util.ScanTaskUtil; /** @@ -43,8 +41,7 @@ *

Readers consuming these tasks should produce added records with metadata like change ordinal * and commit snapshot ID. */ -public interface AddedRowsScanTask - extends ChangelogScanTask, ContentScanTask, HasReferencedFiles { +public interface AddedRowsScanTask extends ChangelogScanTask, ContentScanTask { /** * A list of {@link DeleteFile delete files} to apply when reading the data file in this task. * @@ -66,9 +63,4 @@ default long sizeBytes() { default int filesCount() { return 1 + deletes().size(); } - - @Override - default Iterable> referencedFiles() { - return Iterables.concat(ImmutableList.of(file()), deletes()); - } } diff --git a/api/src/main/java/org/apache/iceberg/DeletedDataFileScanTask.java b/api/src/main/java/org/apache/iceberg/DeletedDataFileScanTask.java index abcf24f443ec..4b9c1704b9d2 100644 --- a/api/src/main/java/org/apache/iceberg/DeletedDataFileScanTask.java +++ b/api/src/main/java/org/apache/iceberg/DeletedDataFileScanTask.java @@ -19,8 +19,6 @@ package org.apache.iceberg; import java.util.List; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; -import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.util.ScanTaskUtil; /** @@ -41,8 +39,7 @@ *

Readers consuming these tasks should produce deleted records with metadata like change ordinal * and commit snapshot ID. */ -public interface DeletedDataFileScanTask - extends ChangelogScanTask, ContentScanTask, HasReferencedFiles { +public interface DeletedDataFileScanTask extends ChangelogScanTask, ContentScanTask { /** * A list of previously added {@link DeleteFile delete files} to apply when reading the data file * in this task. @@ -65,9 +62,4 @@ default long sizeBytes() { default int filesCount() { return 1 + existingDeletes().size(); } - - @Override - default Iterable> referencedFiles() { - return Iterables.concat(ImmutableList.of(file()), existingDeletes()); - } } diff --git a/api/src/main/java/org/apache/iceberg/DeletedRowsScanTask.java b/api/src/main/java/org/apache/iceberg/DeletedRowsScanTask.java index 61b6115e042c..1e0a52a53241 100644 --- a/api/src/main/java/org/apache/iceberg/DeletedRowsScanTask.java +++ b/api/src/main/java/org/apache/iceberg/DeletedRowsScanTask.java @@ -19,8 +19,6 @@ package org.apache.iceberg; import java.util.List; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; -import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.util.ScanTaskUtil; /** @@ -41,8 +39,7 @@ *

Readers consuming these tasks should produce deleted records with metadata like change ordinal * and commit snapshot ID. */ -public interface DeletedRowsScanTask - extends ChangelogScanTask, ContentScanTask, HasReferencedFiles { +public interface DeletedRowsScanTask extends ChangelogScanTask, ContentScanTask { /** * A list of added {@link DeleteFile delete files} that apply to the task's data file. Records * removed by these delete files should appear as deletes in the changelog. @@ -76,9 +73,4 @@ default long sizeBytes() { default int filesCount() { return 1 + addedDeletes().size() + existingDeletes().size(); } - - @Override - default Iterable> referencedFiles() { - return Iterables.concat(ImmutableList.of(file()), addedDeletes(), existingDeletes()); - } } diff --git a/api/src/main/java/org/apache/iceberg/FileScanTask.java b/api/src/main/java/org/apache/iceberg/FileScanTask.java index 0c1169d3ecb7..94f153e56052 100644 --- a/api/src/main/java/org/apache/iceberg/FileScanTask.java +++ b/api/src/main/java/org/apache/iceberg/FileScanTask.java @@ -19,13 +19,10 @@ package org.apache.iceberg; import java.util.List; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; -import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.util.ScanTaskUtil; /** A scan task over a range of bytes in a single data file. */ -public interface FileScanTask - extends ContentScanTask, SplittableScanTask, HasReferencedFiles { +public interface FileScanTask extends ContentScanTask, SplittableScanTask { /** * A list of {@link DeleteFile delete files} to apply when reading the task's data file. * @@ -48,11 +45,6 @@ default int filesCount() { return 1 + deletes().size(); } - @Override - default Iterable> referencedFiles() { - return Iterables.concat(ImmutableList.of(file()), deletes()); - } - @Override default boolean isFileScanTask() { return true; diff --git a/api/src/main/java/org/apache/iceberg/HasReferencedFiles.java b/api/src/main/java/org/apache/iceberg/HasReferencedFiles.java deleted file mode 100644 index 685fc23a772e..000000000000 --- a/api/src/main/java/org/apache/iceberg/HasReferencedFiles.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg; - -/** An object that references content files. */ -public interface HasReferencedFiles { - /** - * Returns the content files referenced by this object. - * - * @return the referenced content files - */ - Iterable> referencedFiles(); -} diff --git a/core/src/main/java/org/apache/iceberg/PositionDeletesScanTask.java b/core/src/main/java/org/apache/iceberg/PositionDeletesScanTask.java index b71ed43e5236..e8ae8a08cd68 100644 --- a/core/src/main/java/org/apache/iceberg/PositionDeletesScanTask.java +++ b/core/src/main/java/org/apache/iceberg/PositionDeletesScanTask.java @@ -18,12 +18,5 @@ */ package org.apache.iceberg; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; - /** A {@link ScanTask} for position delete files */ -public interface PositionDeletesScanTask extends ContentScanTask, HasReferencedFiles { - @Override - default Iterable> referencedFiles() { - return ImmutableList.of(file()); - } -} +public interface PositionDeletesScanTask extends ContentScanTask {} diff --git a/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java b/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java index 75ba8c8f84b4..60d6410cd4c5 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java +++ b/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java @@ -23,12 +23,12 @@ import org.apache.iceberg.CombinedScanTask; import org.apache.iceberg.ContentFile; import org.apache.iceberg.FileScanTask; -import org.apache.iceberg.HasReferencedFiles; import org.apache.iceberg.ScanTask; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.util.TableScanUtil; /** Resolves the files referenced by scan tasks into readable, decrypted {@link InputFile}s. */ public class InputFilesDecryptor { @@ -81,13 +81,9 @@ private Map inputFiles() { private static Collection> referencedFiles(Iterable tasks) { Map> files = Maps.newHashMap(); for (ScanTask task : tasks) { - Preconditions.checkArgument( - task instanceof HasReferencedFiles, - "Unsupported scan task type: %s", - task.getClass().getName()); - ((HasReferencedFiles) task) - .referencedFiles() - .forEach(file -> files.put(file.location(), file)); + for (ContentFile file : TableScanUtil.referencedFiles(task)) { + files.put(file.location(), file); + } } return files.values(); diff --git a/core/src/main/java/org/apache/iceberg/util/TableScanUtil.java b/core/src/main/java/org/apache/iceberg/util/TableScanUtil.java index f143a300b87c..d2862aaaad40 100644 --- a/core/src/main/java/org/apache/iceberg/util/TableScanUtil.java +++ b/core/src/main/java/org/apache/iceberg/util/TableScanUtil.java @@ -22,14 +22,19 @@ import java.util.List; import java.util.Map; import java.util.function.Function; +import org.apache.iceberg.AddedRowsScanTask; import org.apache.iceberg.BaseCombinedScanTask; import org.apache.iceberg.BaseScanTaskGroup; import org.apache.iceberg.CombinedScanTask; +import org.apache.iceberg.ContentFile; +import org.apache.iceberg.DeletedDataFileScanTask; +import org.apache.iceberg.DeletedRowsScanTask; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.MergeableScanTask; import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionScanTask; import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.PositionDeletesScanTask; import org.apache.iceberg.ScanTask; import org.apache.iceberg.ScanTaskGroup; import org.apache.iceberg.SplittableScanTask; @@ -54,6 +59,35 @@ public static boolean hasDeletes(FileScanTask task) { return !task.deletes().isEmpty(); } + public static List> referencedFiles(ScanTask task) { + if (task instanceof FileScanTask fileTask) { + return ImmutableList.>builder() + .add(fileTask.file()) + .addAll(fileTask.deletes()) + .build(); + } else if (task instanceof AddedRowsScanTask addedRowsTask) { + return ImmutableList.>builder() + .add(addedRowsTask.file()) + .addAll(addedRowsTask.deletes()) + .build(); + } else if (task instanceof DeletedDataFileScanTask deletedDataFileTask) { + return ImmutableList.>builder() + .add(deletedDataFileTask.file()) + .addAll(deletedDataFileTask.existingDeletes()) + .build(); + } else if (task instanceof DeletedRowsScanTask deletedRowsTask) { + return ImmutableList.>builder() + .add(deletedRowsTask.file()) + .addAll(deletedRowsTask.addedDeletes()) + .addAll(deletedRowsTask.existingDeletes()) + .build(); + } else if (task instanceof PositionDeletesScanTask positionDeletesTask) { + return ImmutableList.of(positionDeletesTask.file()); + } + + throw new IllegalArgumentException("Unsupported scan task type: " + task.getClass().getName()); + } + public static CloseableIterable splitFiles( CloseableIterable tasks, long splitSize) { Preconditions.checkArgument(splitSize > 0, "Split size must be > 0: %s", splitSize); diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java index 8367315caaa4..9af4c86c97e4 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java @@ -30,7 +30,6 @@ import org.apache.iceberg.ContentFile; import org.apache.iceberg.ContentScanTask; import org.apache.iceberg.DeleteFile; -import org.apache.iceberg.HasReferencedFiles; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.Partitioning; import org.apache.iceberg.ScanTask; @@ -50,13 +49,13 @@ import org.apache.iceberg.io.InputFile; import org.apache.iceberg.mapping.NameMapping; import org.apache.iceberg.mapping.NameMappingParser; -import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.spark.SparkExecutorCache; import org.apache.iceberg.spark.SparkSchemaUtil; import org.apache.iceberg.spark.SparkUtil; import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.StructType; import org.apache.iceberg.util.PartitionUtil; +import org.apache.iceberg.util.TableScanUtil; import org.apache.spark.rdd.InputFileBlockHolder; import org.apache.spark.sql.catalyst.InternalRow; import org.slf4j.Logger; @@ -147,10 +146,11 @@ public boolean next() throws IOException { } } } catch (IOException | RuntimeException e) { - if (currentTask instanceof HasReferencedFiles referencing && !currentTask.isDataTask()) { + if (currentTask != null && !currentTask.isDataTask()) { String filePaths = - String.join( - ", ", Iterables.transform(referencing.referencedFiles(), ContentFile::location)); + TableScanUtil.referencedFiles(currentTask).stream() + .map(ContentFile::location) + .collect(Collectors.joining(", ")); LOG.error("Error reading file(s): {}", filePaths, e); } throw e;