Skip to content
Open
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,13 @@
import java.util.Map;
import org.apache.iceberg.CombinedScanTask;
import org.apache.iceberg.ContentFile;
import org.apache.iceberg.DeleteFile;
import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.ScanTask;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.io.InputFile;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.util.TableScanUtil;

/** Resolves the files referenced by scan tasks into readable, decrypted {@link InputFile}s. */
public class InputFilesDecryptor {
Expand All @@ -48,8 +49,14 @@ public InputFilesDecryptor(
EncryptingFileIO.combine(io, encryption));
}

/**
* Returns a decryptor for the files referenced by the given tasks.
*
* <p>Resolution of referenced files are performed lazily on the first call of {@link
* #getInputFile(String)}.
Comment on lines +55 to +56

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
* <p>Resolution of referenced files are performed lazily on the first call of {@link
* #getInputFile(String)}.
* <p>Referenced files are resolved on the first call to {@link #getInputFile(String)}.

*/
public static InputFilesDecryptor fromTasks(

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we add a short Javadoc here? An unsupported task only fails on the first getInputFile call, which isn't obvious from the signature.

Suggested change
public static InputFilesDecryptor fromTasks(
/**
* Returns a decryptor for the files referenced by the given tasks.
*
* <p>Files are resolved on the first call to {@link #getInputFile(String)}, which throws {@link
* IllegalArgumentException} if a task does not implement {@link HasReferencedFiles}.
*/
public static InputFilesDecryptor fromTasks(

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ahh, I always forget to add the javadoc comment to this public method, thanks for pointing that out and for the suggestion.
I applied it, except the exception part because that seemed to much of an implementation detail to me to leak through Javadoc.

Iterable<FileScanTask> tasks, EncryptingFileIO encryptingIO) {
Iterable<? extends ScanTask> tasks, EncryptingFileIO encryptingIO) {
return new InputFilesDecryptor(() -> referencedFiles(tasks).iterator(), encryptingIO);
}

Expand All @@ -71,12 +78,11 @@ private Map<String, InputFile> inputFiles() {
return lazyInputFiles;
}

private static Collection<ContentFile<?>> referencedFiles(Iterable<FileScanTask> tasks) {
private static Collection<ContentFile<?>> referencedFiles(Iterable<? extends ScanTask> tasks) {
Map<String, ContentFile<?>> files = Maps.newHashMap();
for (FileScanTask task : tasks) {
files.put(task.file().location(), task.file());
for (DeleteFile delete : task.deletes()) {
files.put(delete.location(), delete);
for (ScanTask task : tasks) {
Comment thread
gaborkaszab marked this conversation as resolved.
Resolved
for (ContentFile<?> file : TableScanUtil.referencedFiles(task)) {
files.put(file.location(), file);
}
}

Expand Down
34 changes: 34 additions & 0 deletions core/src/main/java/org/apache/iceberg/util/TableScanUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,19 @@
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import org.apache.iceberg.AddedRowsScanTask;
import org.apache.iceberg.BaseCombinedScanTask;
import org.apache.iceberg.BaseScanTaskGroup;
import org.apache.iceberg.CombinedScanTask;
import org.apache.iceberg.ContentFile;
import org.apache.iceberg.DeletedDataFileScanTask;
import org.apache.iceberg.DeletedRowsScanTask;
import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.MergeableScanTask;
import org.apache.iceberg.PartitionData;
import org.apache.iceberg.PartitionScanTask;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.PositionDeletesScanTask;
import org.apache.iceberg.ScanTask;
import org.apache.iceberg.ScanTaskGroup;
import org.apache.iceberg.SplittableScanTask;
Expand All @@ -54,6 +59,35 @@ public static boolean hasDeletes(FileScanTask task) {
return !task.deletes().isEmpty();
}

public static List<ContentFile<?>> referencedFiles(ScanTask task) {
if (task instanceof FileScanTask fileTask) {
return ImmutableList.<ContentFile<?>>builder()
.add(fileTask.file())
.addAll(fileTask.deletes())
.build();
} else if (task instanceof AddedRowsScanTask addedRowsTask) {
return ImmutableList.<ContentFile<?>>builder()
.add(addedRowsTask.file())
.addAll(addedRowsTask.deletes())
.build();
} else if (task instanceof DeletedDataFileScanTask deletedDataFileTask) {
return ImmutableList.<ContentFile<?>>builder()
.add(deletedDataFileTask.file())
.addAll(deletedDataFileTask.existingDeletes())
.build();
} else if (task instanceof DeletedRowsScanTask deletedRowsTask) {
return ImmutableList.<ContentFile<?>>builder()
.add(deletedRowsTask.file())
.addAll(deletedRowsTask.addedDeletes())
.addAll(deletedRowsTask.existingDeletes())
.build();
} else if (task instanceof PositionDeletesScanTask positionDeletesTask) {
return ImmutableList.of(positionDeletesTask.file());
}

throw new IllegalArgumentException("Unsupported scan task type: " + task.getClass().getName());
}

public static CloseableIterable<FileScanTask> splitFiles(
CloseableIterable<FileScanTask> tasks, long splitSize) {
Preconditions.checkArgument(splitSize > 0, "Split size must be > 0: %s", splitSize);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,18 +24,24 @@
import static org.mockito.ArgumentMatchers.any;

import java.util.List;
import java.util.Map;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.iceberg.AddedRowsScanTask;
import org.apache.iceberg.ContentFile;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.DataFiles;
import org.apache.iceberg.DeleteFile;
import org.apache.iceberg.DeletedDataFileScanTask;
import org.apache.iceberg.DeletedRowsScanTask;
import org.apache.iceberg.FileFormat;
import org.apache.iceberg.FileMetadata;
import org.apache.iceberg.MockFileScanTask;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.PositionDeletesScanTask;
import org.apache.iceberg.ScanTask;
import org.apache.iceberg.io.InputFile;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
Expand Down Expand Up @@ -74,20 +80,48 @@ class TestInputFilesDecryptor {
.withFormat(FileFormat.PARQUET)
.build();

private static final DeleteFile OTHER_DELETE_FILE =
FileMetadata.deleteFileBuilder(SPEC)
.ofPositionDeletes()
.withPath("/path/to/other-deletes.parquet")
.withFileSizeInBytes(54L)
.withRecordCount(2L)
.withFormat(FileFormat.PARQUET)
.build();

private EncryptingFileIO encryptingIO;
private InputFile dataInputFile;
private InputFile deleteInputFile;
private InputFile otherDataInputFile;
private InputFile otherDeleteInputFile;
private Map<String, InputFile> inputFiles;

@BeforeEach
void before() {
this.dataInputFile = Mockito.mock(InputFile.class);
this.deleteInputFile = Mockito.mock(InputFile.class);
this.otherDataInputFile = Mockito.mock(InputFile.class);
this.otherDeleteInputFile = Mockito.mock(InputFile.class);
this.inputFiles =
ImmutableMap.of(
DATA_FILE.location(),
dataInputFile,
DELETE_FILE.location(),
deleteInputFile,
OTHER_DATA_FILE.location(),
otherDataInputFile,
OTHER_DELETE_FILE.location(),
otherDeleteInputFile);
this.encryptingIO = Mockito.mock(EncryptingFileIO.class);
Mockito.when(encryptingIO.bulkDecrypt(any()))
.thenReturn(
ImmutableMap.of(
DATA_FILE.location(), dataInputFile,
DELETE_FILE.location(), deleteInputFile));
.thenAnswer(
invocation -> {
Iterable<ContentFile<?>> files = invocation.getArgument(0);
ImmutableMap.Builder<String, InputFile> decrypted = ImmutableMap.builder();
files.forEach(
file -> decrypted.put(file.location(), inputFiles.get(file.location())));
return decrypted.buildOrThrow();
});
}

@Test
Expand Down Expand Up @@ -143,12 +177,13 @@ void deduplicatesFiles() {

@Test
void resolvesInputFilesOnceWhenAccessedConcurrently() throws Exception {
Mockito.when(encryptingIO.bulkDecrypt(any()))
.thenAnswer(
Mockito.doAnswer(
invocation -> {
Thread.sleep(50);
return ImmutableMap.of(DATA_FILE.location(), dataInputFile);
});
})
.when(encryptingIO)
.bulkDecrypt(any());

InputFilesDecryptor decryptor =
InputFilesDecryptor.fromTasks(
Expand Down Expand Up @@ -177,4 +212,78 @@ void resolvesInputFilesOnceWhenAccessedConcurrently() throws Exception {

Mockito.verify(encryptingIO, Mockito.times(1)).bulkDecrypt(any());
}

@Test
void resolvesFilesOfAddedRowsTasks() {
AddedRowsScanTask task = Mockito.mock(AddedRowsScanTask.class, Mockito.CALLS_REAL_METHODS);
Mockito.doReturn(DATA_FILE).when(task).file();
Mockito.doReturn(ImmutableList.of(DELETE_FILE)).when(task).deletes();

InputFilesDecryptor decryptor =
InputFilesDecryptor.fromTasks(ImmutableList.of(task), encryptingIO);

assertThat(decryptor.getInputFile(DATA_FILE.location())).isSameAs(dataInputFile);
assertThat(decryptor.getInputFile(DELETE_FILE.location())).isSameAs(deleteInputFile);

Mockito.verify(encryptingIO, Mockito.times(1)).bulkDecrypt(any());
}

@Test
void resolvesFilesOfDeletedDataFileTasks() {
DeletedDataFileScanTask task =
Mockito.mock(DeletedDataFileScanTask.class, Mockito.CALLS_REAL_METHODS);
Mockito.doReturn(DATA_FILE).when(task).file();
Mockito.doReturn(ImmutableList.of(DELETE_FILE)).when(task).existingDeletes();

InputFilesDecryptor decryptor =
InputFilesDecryptor.fromTasks(ImmutableList.of(task), encryptingIO);

assertThat(decryptor.getInputFile(DATA_FILE.location())).isSameAs(dataInputFile);
assertThat(decryptor.getInputFile(DELETE_FILE.location())).isSameAs(deleteInputFile);

Mockito.verify(encryptingIO, Mockito.times(1)).bulkDecrypt(any());
}

@Test
void resolvesFilesOfDeletedRowsTasks() {
DeletedRowsScanTask task = Mockito.mock(DeletedRowsScanTask.class, Mockito.CALLS_REAL_METHODS);
Mockito.doReturn(DATA_FILE).when(task).file();
Mockito.doReturn(ImmutableList.of(DELETE_FILE)).when(task).addedDeletes();
Mockito.doReturn(ImmutableList.of(OTHER_DELETE_FILE)).when(task).existingDeletes();

InputFilesDecryptor decryptor =
InputFilesDecryptor.fromTasks(ImmutableList.of(task), encryptingIO);

assertThat(decryptor.getInputFile(DATA_FILE.location())).isSameAs(dataInputFile);
assertThat(decryptor.getInputFile(DELETE_FILE.location())).isSameAs(deleteInputFile);
assertThat(decryptor.getInputFile(OTHER_DELETE_FILE.location())).isSameAs(otherDeleteInputFile);

Mockito.verify(encryptingIO, Mockito.times(1)).bulkDecrypt(any());
}

@Test
void resolvesFilesOfPositionDeletesTasks() {
PositionDeletesScanTask task =
Mockito.mock(PositionDeletesScanTask.class, Mockito.CALLS_REAL_METHODS);
Mockito.doReturn(DELETE_FILE).when(task).file();

InputFilesDecryptor decryptor =
InputFilesDecryptor.fromTasks(ImmutableList.of(task), encryptingIO);

assertThat(decryptor.getInputFile(DELETE_FILE.location())).isSameAs(deleteInputFile);

Mockito.verify(encryptingIO, Mockito.times(1)).bulkDecrypt(any());
}

@Test
void failsForUnsupportedTasks() {
ScanTask task = Mockito.mock(ScanTask.class);

InputFilesDecryptor decryptor =
InputFilesDecryptor.fromTasks(ImmutableList.of(task), encryptingIO);

assertThatThrownBy(() -> decryptor.getInputFile(DATA_FILE.location()))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Unsupported scan task type: " + task.getClass().getName());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.iceberg.ContentFile;
import org.apache.iceberg.ContentScanTask;
import org.apache.iceberg.DeleteFile;
Expand All @@ -44,6 +43,7 @@
import org.apache.iceberg.data.DeleteLoader;
import org.apache.iceberg.deletes.DeleteCounter;
import org.apache.iceberg.encryption.EncryptingFileIO;
import org.apache.iceberg.encryption.InputFilesDecryptor;
import org.apache.iceberg.io.CloseableIterator;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.io.InputFile;
Expand All @@ -55,6 +55,7 @@
import org.apache.iceberg.types.Types;
import org.apache.iceberg.types.Types.StructType;
import org.apache.iceberg.util.PartitionUtil;
import org.apache.iceberg.util.TableScanUtil;
import org.apache.spark.rdd.InputFileBlockHolder;
import org.apache.spark.sql.catalyst.InternalRow;
import org.slf4j.Logger;
Expand All @@ -69,16 +70,14 @@ abstract class BaseReader<T, TaskT extends ScanTask> implements Closeable {
private static final Logger LOG = LoggerFactory.getLogger(BaseReader.class);

private final Table table;
private final EncryptingFileIO fileIO;
private final Schema expectedSchema;
private final boolean caseSensitive;
private final NameMapping nameMapping;
private final ScanTaskGroup<TaskT> taskGroup;
private final Iterator<TaskT> tasks;
private final DeleteCounter counter;
private final boolean cacheDeleteFilesOnExecutors;

private Map<String, InputFile> lazyInputFiles;
private final InputFilesDecryptor inputFilesDecryptor;
private CloseableIterator<T> currentIterator;
private T current = null;
private TaskT currentTask = null;
Expand All @@ -91,8 +90,6 @@ abstract class BaseReader<T, TaskT extends ScanTask> implements Closeable {
boolean caseSensitive,
boolean cacheDeleteFilesOnExecutors) {
this.table = table;
this.fileIO = EncryptingFileIO.combine(fileIO, table().encryption());
this.taskGroup = taskGroup;
this.tasks = taskGroup.tasks().iterator();
this.currentIterator = CloseableIterator.empty();
this.expectedSchema = expectedSchema;
Expand All @@ -102,12 +99,13 @@ abstract class BaseReader<T, TaskT extends ScanTask> implements Closeable {
nameMappingString != null ? NameMappingParser.fromJson(nameMappingString) : null;
this.counter = new DeleteCounter();
this.cacheDeleteFilesOnExecutors = cacheDeleteFilesOnExecutors;
this.inputFilesDecryptor =
InputFilesDecryptor.fromTasks(
taskGroup.tasks(), EncryptingFileIO.combine(fileIO, table.encryption()));
}

protected abstract CloseableIterator<T> open(TaskT task);

protected abstract Stream<ContentFile<?>> referencedFiles(TaskT task);

protected Schema expectedSchema() {
return expectedSchema;
}
Expand Down Expand Up @@ -150,7 +148,7 @@ public boolean next() throws IOException {
} catch (IOException | RuntimeException e) {
if (currentTask != null && !currentTask.isDataTask()) {
String filePaths =
referencedFiles(currentTask)
TableScanUtil.referencedFiles(currentTask).stream()
.map(ContentFile::location)
.collect(Collectors.joining(", "));
LOG.error("Error reading file(s): {}", filePaths, e);
Expand All @@ -177,17 +175,7 @@ public void close() throws IOException {
}

protected InputFile getInputFile(String location) {
return inputFiles().get(location);
}

private Map<String, InputFile> inputFiles() {
if (lazyInputFiles == null) {
this.lazyInputFiles =
fileIO.bulkDecrypt(
() -> taskGroup.tasks().stream().flatMap(this::referencedFiles).iterator());
}

return lazyInputFiles;
return inputFilesDecryptor.getInputFile(location);
}

protected Map<Integer, ?> constantsMap(ContentScanTask<?> task, Schema readSchema) {
Expand Down
Loading
Loading