diff --git a/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java b/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java index 6245fa119191..60d6410cd4c5 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java +++ b/core/src/main/java/org/apache/iceberg/encryption/InputFilesDecryptor.java @@ -22,12 +22,13 @@ import java.util.Map; import org.apache.iceberg.CombinedScanTask; import org.apache.iceberg.ContentFile; -import org.apache.iceberg.DeleteFile; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.ScanTask; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.util.TableScanUtil; /** Resolves the files referenced by scan tasks into readable, decrypted {@link InputFile}s. */ public class InputFilesDecryptor { @@ -48,8 +49,14 @@ public InputFilesDecryptor( EncryptingFileIO.combine(io, encryption)); } + /** + * Returns a decryptor for the files referenced by the given tasks. + * + *

Resolution of referenced files are performed lazily on the first call of {@link + * #getInputFile(String)}. + */ public static InputFilesDecryptor fromTasks( - Iterable tasks, EncryptingFileIO encryptingIO) { + Iterable tasks, EncryptingFileIO encryptingIO) { return new InputFilesDecryptor(() -> referencedFiles(tasks).iterator(), encryptingIO); } @@ -71,12 +78,11 @@ private Map inputFiles() { return lazyInputFiles; } - private static Collection> referencedFiles(Iterable tasks) { + private static Collection> referencedFiles(Iterable tasks) { Map> files = Maps.newHashMap(); - for (FileScanTask task : tasks) { - files.put(task.file().location(), task.file()); - for (DeleteFile delete : task.deletes()) { - files.put(delete.location(), delete); + for (ScanTask task : tasks) { + for (ContentFile file : TableScanUtil.referencedFiles(task)) { + files.put(file.location(), file); } } diff --git a/core/src/main/java/org/apache/iceberg/util/TableScanUtil.java b/core/src/main/java/org/apache/iceberg/util/TableScanUtil.java index f143a300b87c..d2862aaaad40 100644 --- a/core/src/main/java/org/apache/iceberg/util/TableScanUtil.java +++ b/core/src/main/java/org/apache/iceberg/util/TableScanUtil.java @@ -22,14 +22,19 @@ import java.util.List; import java.util.Map; import java.util.function.Function; +import org.apache.iceberg.AddedRowsScanTask; import org.apache.iceberg.BaseCombinedScanTask; import org.apache.iceberg.BaseScanTaskGroup; import org.apache.iceberg.CombinedScanTask; +import org.apache.iceberg.ContentFile; +import org.apache.iceberg.DeletedDataFileScanTask; +import org.apache.iceberg.DeletedRowsScanTask; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.MergeableScanTask; import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionScanTask; import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.PositionDeletesScanTask; import org.apache.iceberg.ScanTask; import org.apache.iceberg.ScanTaskGroup; import org.apache.iceberg.SplittableScanTask; @@ -54,6 +59,35 @@ public static boolean hasDeletes(FileScanTask task) { return !task.deletes().isEmpty(); } + public static List> referencedFiles(ScanTask task) { + if (task instanceof FileScanTask fileTask) { + return ImmutableList.>builder() + .add(fileTask.file()) + .addAll(fileTask.deletes()) + .build(); + } else if (task instanceof AddedRowsScanTask addedRowsTask) { + return ImmutableList.>builder() + .add(addedRowsTask.file()) + .addAll(addedRowsTask.deletes()) + .build(); + } else if (task instanceof DeletedDataFileScanTask deletedDataFileTask) { + return ImmutableList.>builder() + .add(deletedDataFileTask.file()) + .addAll(deletedDataFileTask.existingDeletes()) + .build(); + } else if (task instanceof DeletedRowsScanTask deletedRowsTask) { + return ImmutableList.>builder() + .add(deletedRowsTask.file()) + .addAll(deletedRowsTask.addedDeletes()) + .addAll(deletedRowsTask.existingDeletes()) + .build(); + } else if (task instanceof PositionDeletesScanTask positionDeletesTask) { + return ImmutableList.of(positionDeletesTask.file()); + } + + throw new IllegalArgumentException("Unsupported scan task type: " + task.getClass().getName()); + } + public static CloseableIterable splitFiles( CloseableIterable tasks, long splitSize) { Preconditions.checkArgument(splitSize > 0, "Split size must be > 0: %s", splitSize); diff --git a/core/src/test/java/org/apache/iceberg/encryption/TestInputFilesDecryptor.java b/core/src/test/java/org/apache/iceberg/encryption/TestInputFilesDecryptor.java index a0b82ba61e3f..0e91d77d081b 100644 --- a/core/src/test/java/org/apache/iceberg/encryption/TestInputFilesDecryptor.java +++ b/core/src/test/java/org/apache/iceberg/encryption/TestInputFilesDecryptor.java @@ -24,18 +24,24 @@ import static org.mockito.ArgumentMatchers.any; import java.util.List; +import java.util.Map; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import org.apache.iceberg.AddedRowsScanTask; import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.DeletedDataFileScanTask; +import org.apache.iceberg.DeletedRowsScanTask; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileMetadata; import org.apache.iceberg.MockFileScanTask; import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.PositionDeletesScanTask; +import org.apache.iceberg.ScanTask; import org.apache.iceberg.io.InputFile; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; @@ -74,20 +80,48 @@ class TestInputFilesDecryptor { .withFormat(FileFormat.PARQUET) .build(); + private static final DeleteFile OTHER_DELETE_FILE = + FileMetadata.deleteFileBuilder(SPEC) + .ofPositionDeletes() + .withPath("/path/to/other-deletes.parquet") + .withFileSizeInBytes(54L) + .withRecordCount(2L) + .withFormat(FileFormat.PARQUET) + .build(); + private EncryptingFileIO encryptingIO; private InputFile dataInputFile; private InputFile deleteInputFile; + private InputFile otherDataInputFile; + private InputFile otherDeleteInputFile; + private Map inputFiles; @BeforeEach void before() { this.dataInputFile = Mockito.mock(InputFile.class); this.deleteInputFile = Mockito.mock(InputFile.class); + this.otherDataInputFile = Mockito.mock(InputFile.class); + this.otherDeleteInputFile = Mockito.mock(InputFile.class); + this.inputFiles = + ImmutableMap.of( + DATA_FILE.location(), + dataInputFile, + DELETE_FILE.location(), + deleteInputFile, + OTHER_DATA_FILE.location(), + otherDataInputFile, + OTHER_DELETE_FILE.location(), + otherDeleteInputFile); this.encryptingIO = Mockito.mock(EncryptingFileIO.class); Mockito.when(encryptingIO.bulkDecrypt(any())) - .thenReturn( - ImmutableMap.of( - DATA_FILE.location(), dataInputFile, - DELETE_FILE.location(), deleteInputFile)); + .thenAnswer( + invocation -> { + Iterable> files = invocation.getArgument(0); + ImmutableMap.Builder decrypted = ImmutableMap.builder(); + files.forEach( + file -> decrypted.put(file.location(), inputFiles.get(file.location()))); + return decrypted.buildOrThrow(); + }); } @Test @@ -143,12 +177,13 @@ void deduplicatesFiles() { @Test void resolvesInputFilesOnceWhenAccessedConcurrently() throws Exception { - Mockito.when(encryptingIO.bulkDecrypt(any())) - .thenAnswer( + Mockito.doAnswer( invocation -> { Thread.sleep(50); return ImmutableMap.of(DATA_FILE.location(), dataInputFile); - }); + }) + .when(encryptingIO) + .bulkDecrypt(any()); InputFilesDecryptor decryptor = InputFilesDecryptor.fromTasks( @@ -177,4 +212,78 @@ void resolvesInputFilesOnceWhenAccessedConcurrently() throws Exception { Mockito.verify(encryptingIO, Mockito.times(1)).bulkDecrypt(any()); } + + @Test + void resolvesFilesOfAddedRowsTasks() { + AddedRowsScanTask task = Mockito.mock(AddedRowsScanTask.class, Mockito.CALLS_REAL_METHODS); + Mockito.doReturn(DATA_FILE).when(task).file(); + Mockito.doReturn(ImmutableList.of(DELETE_FILE)).when(task).deletes(); + + InputFilesDecryptor decryptor = + InputFilesDecryptor.fromTasks(ImmutableList.of(task), encryptingIO); + + assertThat(decryptor.getInputFile(DATA_FILE.location())).isSameAs(dataInputFile); + assertThat(decryptor.getInputFile(DELETE_FILE.location())).isSameAs(deleteInputFile); + + Mockito.verify(encryptingIO, Mockito.times(1)).bulkDecrypt(any()); + } + + @Test + void resolvesFilesOfDeletedDataFileTasks() { + DeletedDataFileScanTask task = + Mockito.mock(DeletedDataFileScanTask.class, Mockito.CALLS_REAL_METHODS); + Mockito.doReturn(DATA_FILE).when(task).file(); + Mockito.doReturn(ImmutableList.of(DELETE_FILE)).when(task).existingDeletes(); + + InputFilesDecryptor decryptor = + InputFilesDecryptor.fromTasks(ImmutableList.of(task), encryptingIO); + + assertThat(decryptor.getInputFile(DATA_FILE.location())).isSameAs(dataInputFile); + assertThat(decryptor.getInputFile(DELETE_FILE.location())).isSameAs(deleteInputFile); + + Mockito.verify(encryptingIO, Mockito.times(1)).bulkDecrypt(any()); + } + + @Test + void resolvesFilesOfDeletedRowsTasks() { + DeletedRowsScanTask task = Mockito.mock(DeletedRowsScanTask.class, Mockito.CALLS_REAL_METHODS); + Mockito.doReturn(DATA_FILE).when(task).file(); + Mockito.doReturn(ImmutableList.of(DELETE_FILE)).when(task).addedDeletes(); + Mockito.doReturn(ImmutableList.of(OTHER_DELETE_FILE)).when(task).existingDeletes(); + + InputFilesDecryptor decryptor = + InputFilesDecryptor.fromTasks(ImmutableList.of(task), encryptingIO); + + assertThat(decryptor.getInputFile(DATA_FILE.location())).isSameAs(dataInputFile); + assertThat(decryptor.getInputFile(DELETE_FILE.location())).isSameAs(deleteInputFile); + assertThat(decryptor.getInputFile(OTHER_DELETE_FILE.location())).isSameAs(otherDeleteInputFile); + + Mockito.verify(encryptingIO, Mockito.times(1)).bulkDecrypt(any()); + } + + @Test + void resolvesFilesOfPositionDeletesTasks() { + PositionDeletesScanTask task = + Mockito.mock(PositionDeletesScanTask.class, Mockito.CALLS_REAL_METHODS); + Mockito.doReturn(DELETE_FILE).when(task).file(); + + InputFilesDecryptor decryptor = + InputFilesDecryptor.fromTasks(ImmutableList.of(task), encryptingIO); + + assertThat(decryptor.getInputFile(DELETE_FILE.location())).isSameAs(deleteInputFile); + + Mockito.verify(encryptingIO, Mockito.times(1)).bulkDecrypt(any()); + } + + @Test + void failsForUnsupportedTasks() { + ScanTask task = Mockito.mock(ScanTask.class); + + InputFilesDecryptor decryptor = + InputFilesDecryptor.fromTasks(ImmutableList.of(task), encryptingIO); + + assertThatThrownBy(() -> decryptor.getInputFile(DATA_FILE.location())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unsupported scan task type: " + task.getClass().getName()); + } } diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java index a4d9766ae713..9af4c86c97e4 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java @@ -27,7 +27,6 @@ import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.iceberg.ContentFile; import org.apache.iceberg.ContentScanTask; import org.apache.iceberg.DeleteFile; @@ -44,6 +43,7 @@ import org.apache.iceberg.data.DeleteLoader; import org.apache.iceberg.deletes.DeleteCounter; import org.apache.iceberg.encryption.EncryptingFileIO; +import org.apache.iceberg.encryption.InputFilesDecryptor; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; @@ -55,6 +55,7 @@ import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.StructType; import org.apache.iceberg.util.PartitionUtil; +import org.apache.iceberg.util.TableScanUtil; import org.apache.spark.rdd.InputFileBlockHolder; import org.apache.spark.sql.catalyst.InternalRow; import org.slf4j.Logger; @@ -69,16 +70,14 @@ abstract class BaseReader implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(BaseReader.class); private final Table table; - private final EncryptingFileIO fileIO; private final Schema expectedSchema; private final boolean caseSensitive; private final NameMapping nameMapping; - private final ScanTaskGroup taskGroup; private final Iterator tasks; private final DeleteCounter counter; private final boolean cacheDeleteFilesOnExecutors; - private Map lazyInputFiles; + private final InputFilesDecryptor inputFilesDecryptor; private CloseableIterator currentIterator; private T current = null; private TaskT currentTask = null; @@ -91,8 +90,6 @@ abstract class BaseReader implements Closeable { boolean caseSensitive, boolean cacheDeleteFilesOnExecutors) { this.table = table; - this.fileIO = EncryptingFileIO.combine(fileIO, table().encryption()); - this.taskGroup = taskGroup; this.tasks = taskGroup.tasks().iterator(); this.currentIterator = CloseableIterator.empty(); this.expectedSchema = expectedSchema; @@ -102,12 +99,13 @@ abstract class BaseReader implements Closeable { nameMappingString != null ? NameMappingParser.fromJson(nameMappingString) : null; this.counter = new DeleteCounter(); this.cacheDeleteFilesOnExecutors = cacheDeleteFilesOnExecutors; + this.inputFilesDecryptor = + InputFilesDecryptor.fromTasks( + taskGroup.tasks(), EncryptingFileIO.combine(fileIO, table.encryption())); } protected abstract CloseableIterator open(TaskT task); - protected abstract Stream> referencedFiles(TaskT task); - protected Schema expectedSchema() { return expectedSchema; } @@ -150,7 +148,7 @@ public boolean next() throws IOException { } catch (IOException | RuntimeException e) { if (currentTask != null && !currentTask.isDataTask()) { String filePaths = - referencedFiles(currentTask) + TableScanUtil.referencedFiles(currentTask).stream() .map(ContentFile::location) .collect(Collectors.joining(", ")); LOG.error("Error reading file(s): {}", filePaths, e); @@ -177,17 +175,7 @@ public void close() throws IOException { } protected InputFile getInputFile(String location) { - return inputFiles().get(location); - } - - private Map inputFiles() { - if (lazyInputFiles == null) { - this.lazyInputFiles = - fileIO.bulkDecrypt( - () -> taskGroup.tasks().stream().flatMap(this::referencedFiles).iterator()); - } - - return lazyInputFiles; + return inputFilesDecryptor.getInputFile(location); } protected Map constantsMap(ContentScanTask task, Schema readSchema) { diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BatchDataReader.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BatchDataReader.java index 237dfd5c6919..327ad93753c2 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BatchDataReader.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BatchDataReader.java @@ -19,8 +19,6 @@ package org.apache.iceberg.spark.source; import java.util.Map; -import java.util.stream.Stream; -import org.apache.iceberg.ContentFile; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.ScanTaskGroup; import org.apache.iceberg.Schema; @@ -28,7 +26,6 @@ import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.spark.OrcBatchReadConf; import org.apache.iceberg.spark.ParquetBatchReadConf; import org.apache.iceberg.spark.source.metrics.TaskNumDeletes; @@ -92,11 +89,6 @@ public CustomTaskMetric[] currentMetricsValues() { }; } - @Override - protected Stream> referencedFiles(FileScanTask task) { - return Stream.concat(Stream.of(task.file()), task.deletes().stream()); - } - @Override protected CloseableIterator open(FileScanTask task) { String filePath = task.file().location(); @@ -106,7 +98,6 @@ protected CloseableIterator open(FileScanTask task) { InputFileBlockHolder.set(filePath, task.start(), task.length()); InputFile inputFile = getInputFile(filePath); - Preconditions.checkNotNull(inputFile, "Could not find InputFile associated with FileScanTask"); SparkDeleteFilter deleteFilter = new SparkDeleteFilter(filePath, task.deletes(), counter(), true); diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/ChangelogRowReader.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/ChangelogRowReader.java index eb8e5e63f430..2343ae774768 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/ChangelogRowReader.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/ChangelogRowReader.java @@ -18,16 +18,12 @@ */ package org.apache.iceberg.spark.source; -import java.util.List; import java.util.Map; -import java.util.stream.Stream; import org.apache.iceberg.AddedRowsScanTask; import org.apache.iceberg.ChangelogScanTask; import org.apache.iceberg.ChangelogUtil; -import org.apache.iceberg.ContentFile; import org.apache.iceberg.ContentScanTask; import org.apache.iceberg.DataFile; -import org.apache.iceberg.DeleteFile; import org.apache.iceberg.DeletedDataFileScanTask; import org.apache.iceberg.DeletedRowsScanTask; import org.apache.iceberg.ScanTaskGroup; @@ -37,7 +33,6 @@ import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.spark.rdd.InputFileBlockHolder; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.catalyst.expressions.GenericInternalRow; @@ -134,7 +129,6 @@ private CloseableIterable rows(ContentScanTask task, Sche InputFileBlockHolder.set(filePath, task.start(), task.length()); InputFile location = getInputFile(filePath); - Preconditions.checkNotNull(location, "Could not find InputFile"); return newIterable( location, task.file().format(), @@ -144,33 +138,4 @@ private CloseableIterable rows(ContentScanTask task, Sche readSchema, idToConstant); } - - @Override - protected Stream> referencedFiles(ChangelogScanTask task) { - if (task instanceof AddedRowsScanTask) { - return addedRowsScanTaskFiles((AddedRowsScanTask) task); - - } else if (task instanceof DeletedRowsScanTask) { - throw new UnsupportedOperationException("Deleted rows scan task is not supported yet"); - - } else if (task instanceof DeletedDataFileScanTask) { - return deletedDataFileScanTaskFiles((DeletedDataFileScanTask) task); - - } else { - throw new IllegalArgumentException( - "Unsupported changelog scan task type: " + task.getClass().getName()); - } - } - - private static Stream> deletedDataFileScanTaskFiles(DeletedDataFileScanTask task) { - DataFile file = task.file(); - List existingDeletes = task.existingDeletes(); - return Stream.concat(Stream.of(file), existingDeletes.stream()); - } - - private static Stream> addedRowsScanTaskFiles(AddedRowsScanTask task) { - DataFile file = task.file(); - List deletes = task.deletes(); - return Stream.concat(Stream.of(file), deletes.stream()); - } } diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/PositionDeletesRowReader.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/PositionDeletesRowReader.java index c7abf219db02..06b702fef753 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/PositionDeletesRowReader.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/PositionDeletesRowReader.java @@ -19,8 +19,6 @@ package org.apache.iceberg.spark.source; import java.util.Map; -import java.util.stream.Stream; -import org.apache.iceberg.ContentFile; import org.apache.iceberg.PositionDeletesScanTask; import org.apache.iceberg.ScanTaskGroup; import org.apache.iceberg.Schema; @@ -30,7 +28,6 @@ import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.util.ContentFileUtil; import org.apache.spark.rdd.InputFileBlockHolder; import org.apache.spark.sql.catalyst.InternalRow; @@ -66,11 +63,6 @@ class PositionDeletesRowReader extends BaseRowReader LOG.debug("Reading {} position delete file split(s) for table {}", numSplits, table.name()); } - @Override - protected Stream> referencedFiles(PositionDeletesScanTask task) { - return Stream.of(task.file()); - } - @SuppressWarnings("resource") // handled by BaseReader @Override protected CloseableIterator open(PositionDeletesScanTask task) { @@ -81,7 +73,6 @@ protected CloseableIterator open(PositionDeletesScanTask task) { InputFileBlockHolder.set(filePath, task.start(), task.length()); InputFile inputFile = getInputFile(task.file().location()); - Preconditions.checkNotNull(inputFile, "Could not find InputFile associated with %s", task); // Retain predicates on non-constant fields for row reader filter Map idToConstant = constantsMap(task, expectedSchema()); diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/RowDataReader.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/RowDataReader.java index dbfb0b7614e4..25f60e93154a 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/RowDataReader.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/RowDataReader.java @@ -19,8 +19,6 @@ package org.apache.iceberg.spark.source; import java.util.Map; -import java.util.stream.Stream; -import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataTask; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.ScanTaskGroup; @@ -30,7 +28,6 @@ import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.spark.source.metrics.TaskNumDeletes; import org.apache.iceberg.spark.source.metrics.TaskNumSplits; import org.apache.spark.rdd.InputFileBlockHolder; @@ -76,11 +73,6 @@ public CustomTaskMetric[] currentMetricsValues() { }; } - @Override - protected Stream> referencedFiles(FileScanTask task) { - return Stream.concat(Stream.of(task.file()), task.deletes().stream()); - } - @Override protected CloseableIterator open(FileScanTask task) { String filePath = task.file().location(); @@ -104,8 +96,6 @@ protected CloseableIterable open( return newDataIterable(task.asDataTask(), readSchema); } else { InputFile inputFile = getInputFile(task.file().location()); - Preconditions.checkNotNull( - inputFile, "Could not find InputFile associated with FileScanTask"); return newIterable( inputFile, task.file().format(), diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestBaseReader.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestBaseReader.java index 0eb9bbe52ffa..71bb111e54e8 100644 --- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestBaseReader.java +++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestBaseReader.java @@ -31,12 +31,10 @@ import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.IntStream; -import java.util.stream.Stream; import java.util.stream.StreamSupport; import org.apache.avro.generic.GenericData; import org.apache.iceberg.AppendFiles; import org.apache.iceberg.BaseCombinedScanTask; -import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.FileScanTask; @@ -93,11 +91,6 @@ private static class ClosureTrackingReader extends BaseReader> referencedFiles(FileScanTask task) { - return Stream.of(); - } - @Override protected CloseableIterator open(FileScanTask task) { CloseableIntegerRange intRange = new CloseableIntegerRange(task.file().recordCount());