From 14ef12125040f7969448fbd39f9e34f617447a1e Mon Sep 17 00:00:00 2001 From: Gabor Kaszab Date: Tue, 13 Oct 2026 11:15:30 +0200 Subject: [PATCH 1/3] Data, MR: IcebergInputFormat to read encrypted delete files IcebergInputFormat decrypted data files but passed the plain table FileIO to GenericDeleteFilter, so encrypted delete files were read without decryption. The record reader now resolves both data and delete files through an InputFilesDecryptor. Generated-by: GitHub Copilot CLI (Claude Opus 5.5) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../iceberg/data/DeleteFilterBenchmark.java | 7 +- .../iceberg/data/GenericDeleteFilter.java | 20 ++- .../apache/iceberg/data/GenericReader.java | 2 +- .../mr/mapreduce/IcebergInputFormat.java | 20 +-- .../TestIcebergInputFormatEncryption.java | 160 ++++++++++++++++++ 5 files changed, 193 insertions(+), 16 deletions(-) create mode 100644 mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormatEncryption.java diff --git a/data/src/jmh/java/org/apache/iceberg/data/DeleteFilterBenchmark.java b/data/src/jmh/java/org/apache/iceberg/data/DeleteFilterBenchmark.java index ab918c9902d4..dbbfcab2548c 100644 --- a/data/src/jmh/java/org/apache/iceberg/data/DeleteFilterBenchmark.java +++ b/data/src/jmh/java/org/apache/iceberg/data/DeleteFilterBenchmark.java @@ -34,6 +34,8 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; +import org.apache.iceberg.encryption.EncryptingFileIO; +import org.apache.iceberg.encryption.InputFilesDecryptor; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; @@ -164,7 +166,10 @@ public void setupBenchmark() throws IOException { tasks = Lists.newArrayList(planned); } FileScanTask task = Iterables.getOnlyElement(tasks); - this.filter = new GenericDeleteFilter(table.io(), task, table.schema(), table.schema()); + InputFilesDecryptor decryptor = + InputFilesDecryptor.fromTasks( + tasks, EncryptingFileIO.combine(table.io(), table.encryption())); + this.filter = new GenericDeleteFilter(decryptor, task, table.schema(), table.schema()); this.filterInput = materializeFilterInput(rows, filter.requiredSchema()); } diff --git a/data/src/main/java/org/apache/iceberg/data/GenericDeleteFilter.java b/data/src/main/java/org/apache/iceberg/data/GenericDeleteFilter.java index 55fbe1aff34f..e2b826639235 100644 --- a/data/src/main/java/org/apache/iceberg/data/GenericDeleteFilter.java +++ b/data/src/main/java/org/apache/iceberg/data/GenericDeleteFilter.java @@ -22,6 +22,7 @@ import org.apache.iceberg.FileScanTask; import org.apache.iceberg.Schema; import org.apache.iceberg.StructLike; +import org.apache.iceberg.encryption.InputFilesDecryptor; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; @@ -29,18 +30,31 @@ public class GenericDeleteFilter extends DeleteFilter { private final Function inputFileProvider; private final InternalRecordWrapper asStructLike; + /** + * @deprecated since 1.13.0, will be removed in 1.14.0; use {@link + * #GenericDeleteFilter(InputFilesDecryptor, FileScanTask, Schema, Schema)} instead. + */ + @Deprecated public GenericDeleteFilter( FileIO io, FileScanTask task, Schema tableSchema, Schema requestedSchema) { this(io::newInputFile, task, tableSchema, requestedSchema); } - GenericDeleteFilter( - Function inputFileProvider, + public GenericDeleteFilter( + InputFilesDecryptor decryptor, + FileScanTask task, + Schema tableSchema, + Schema requestedSchema) { + this(decryptor::getInputFile, task, tableSchema, requestedSchema); + } + + private GenericDeleteFilter( + Function inputFiles, FileScanTask task, Schema tableSchema, Schema requestedSchema) { super(task.file().location(), task.deletes(), tableSchema, requestedSchema); - this.inputFileProvider = inputFileProvider; + this.inputFileProvider = inputFiles; this.asStructLike = new InternalRecordWrapper(requiredSchema().asStruct()); } diff --git a/data/src/main/java/org/apache/iceberg/data/GenericReader.java b/data/src/main/java/org/apache/iceberg/data/GenericReader.java index 12ea8d1bec8e..9c5afa4aa46f 100644 --- a/data/src/main/java/org/apache/iceberg/data/GenericReader.java +++ b/data/src/main/java/org/apache/iceberg/data/GenericReader.java @@ -62,7 +62,7 @@ public CloseableIterable open(CombinedScanTask task) { private CloseableIterable open(FileScanTask task, InputFilesDecryptor decryptor) { DeleteFilter deletes = - new GenericDeleteFilter(decryptor::getInputFile, task, tableSchema, projection); + new GenericDeleteFilter(decryptor, task, tableSchema, projection); Schema readSchema = deletes.requiredSchema(); CloseableIterable records = openFile(task, readSchema, decryptor); diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java index 764648e26302..88f8b1329a66 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java @@ -48,8 +48,8 @@ import org.apache.iceberg.data.IdentityPartitionConverters; import org.apache.iceberg.data.InternalRecordWrapper; import org.apache.iceberg.data.Record; -import org.apache.iceberg.encryption.EncryptedFiles; -import org.apache.iceberg.encryption.EncryptionManager; +import org.apache.iceberg.encryption.EncryptingFileIO; +import org.apache.iceberg.encryption.InputFilesDecryptor; import org.apache.iceberg.expressions.Evaluator; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; @@ -58,7 +58,6 @@ import org.apache.iceberg.hadoop.HadoopConfigurable; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.CloseableIterator; -import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; import org.apache.iceberg.mapping.NameMappingParser; import org.apache.iceberg.mr.Catalogs; @@ -236,8 +235,7 @@ private static final class IcebergRecordReader extends RecordReader private Iterator tasks; private T current; private CloseableIterator currentIterator; - private FileIO io; - private EncryptionManager encryptionManager; + private InputFilesDecryptor decryptor; @Override public void initialize(InputSplit split, TaskAttemptContext newContext) { @@ -248,8 +246,9 @@ public void initialize(InputSplit split, TaskAttemptContext newContext) { this.context = newContext; Table table = ((IcebergSplit) split).table(); checkAndSetIoConfig(conf, table); - this.io = table.io(); - this.encryptionManager = table.encryption(); + this.decryptor = + InputFilesDecryptor.fromTasks( + task.files(), EncryptingFileIO.combine(table.io(), table.encryption())); this.tasks = task.files().iterator(); this.tableSchema = InputFormatConfig.tableSchema(conf); this.nameMapping = table.properties().get(TableProperties.DEFAULT_NAME_MAPPING); @@ -311,9 +310,7 @@ public void close() throws IOException { @SuppressWarnings("unchecked") private CloseableIterable openTask(FileScanTask currentTask, Schema readSchema) { DataFile file = currentTask.file(); - InputFile inputFile = - encryptionManager.decrypt( - EncryptedFiles.encryptedInput(io.newInputFile(file.location()), file.keyMetadata())); + InputFile inputFile = decryptor.getInputFile(file.location()); ReadBuilder readBuilder = FormatModelRegistry.readBuilder(file.format(), Record.class, inputFile); @@ -343,7 +340,8 @@ private CloseableIterable openTask(FileScanTask currentTask, Schema readSchem @SuppressWarnings("unchecked") private CloseableIterable open(FileScanTask currentTask, Schema readSchema) { - DeleteFilter deletes = new GenericDeleteFilter(io, currentTask, tableSchema, readSchema); + DeleteFilter deletes = + new GenericDeleteFilter(decryptor, currentTask, tableSchema, readSchema); Schema requiredSchema = deletes.requiredSchema(); return deletes.filter(openTask(currentTask, requiredSchema)); } diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormatEncryption.java b/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormatEncryption.java new file mode 100644 index 000000000000..390d6846b6f5 --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormatEncryption.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.mr.mapreduce; + +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.apache.iceberg.types.Types.NestedField.required; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.TaskAttemptID; +import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; +import org.apache.iceberg.CombinedScanTask; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.Files; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SchemaParser; +import org.apache.iceberg.SerializableTable; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.Table; +import org.apache.iceberg.TestTables; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.encryption.EncryptingFileIO; +import org.apache.iceberg.encryption.EncryptionManager; +import org.apache.iceberg.encryption.EncryptionTestHelpers; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.mr.InputFormatConfig; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.Pair; +import org.apache.iceberg.util.StructLikeSet; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class TestIcebergInputFormatEncryption { + private static final Schema SCHEMA = + new Schema( + required(1, "id", Types.LongType.get()), optional(2, "data", Types.StringType.get())); + + @TempDir private File tableDir; + + private Table table; + + @BeforeEach + void createTable() { + EncryptionManager encryptionManager = EncryptionTestHelpers.createEncryptionManager(); + TestTables.TestTableOperations ops = + new TestTables.TestTableOperations( + "encrypted", + tableDir, + EncryptingFileIO.combine(new TestTables.LocalFileIO(), encryptionManager)) { + @Override + public EncryptionManager encryption() { + return encryptionManager; + } + }; + + this.table = + TestTables.create( + tableDir, + "encrypted", + SCHEMA, + PartitionSpec.unpartitioned(), + SortOrder.unsorted(), + 3, + ops); + } + + @AfterEach + void dropTable() { + TestTables.clearTables(); + } + + @Test + void readEncryptedDataFilesWithDV() throws IOException, InterruptedException { + Record first = record(1L, "a"); + Record second = record(2L, "b"); + Record third = record(3L, "c"); + DataFile firstFile = + FileHelpers.writeDataFile(table, outputFile("first.parquet"), List.of(first, second)); + DataFile secondFile = + FileHelpers.writeDataFile(table, outputFile("second.parquet"), List.of(third)); + table.newAppend().appendFile(firstFile).appendFile(secondFile).commit(); + + DeleteFile dv = + FileHelpers.writeDeleteFile(table, null, List.of(Pair.of(firstFile.location(), 0L)), 3) + .first(); + table.newRowDelta().addDeletes(dv).commit(); + + assertThat(firstFile.keyMetadata()).isNotNull(); + assertThat(secondFile.keyMetadata()).isNotNull(); + assertThat(dv.keyMetadata()).isNotNull(); + + StructLikeSet expected = StructLikeSet.create(SCHEMA.asStruct()); + expected.add(second); + expected.add(third); + + assertThat(readWithInputFormat()).isEqualTo(expected); + } + + private StructLikeSet readWithInputFormat() throws IOException, InterruptedException { + Configuration conf = new Configuration(); + conf.set(InputFormatConfig.TABLE_SCHEMA, SchemaParser.toJson(table.schema())); + TaskAttemptContext context = new TaskAttemptContextImpl(conf, new TaskAttemptID()); + IcebergInputFormat inputFormat = new IcebergInputFormat<>(); + Table serializableTable = SerializableTable.copyOf(table); + + StructLikeSet actual = StructLikeSet.create(SCHEMA.asStruct()); + try (CloseableIterable tasks = table.newScan().planTasks()) { + for (CombinedScanTask task : tasks) { + IcebergSplit split = new IcebergSplit(serializableTable, conf, task); + try (RecordReader reader = inputFormat.createRecordReader(split, context)) { + reader.initialize(split, context); + while (reader.nextKeyValue()) { + actual.add(reader.getCurrentValue()); + } + } + } + } + + return actual; + } + + private static Record record(long id, String data) { + Record record = GenericRecord.create(SCHEMA); + record.setField("id", id); + record.setField("data", data); + return record; + } + + private OutputFile outputFile(String fileName) { + return Files.localOutput(new File(tableDir, fileName)); + } +} From 032d644fc251a32c6aabc7633de3ebb28b607de0 Mon Sep 17 00:00:00 2001 From: Gabor Kaszab Date: Tue, 17 Nov 2026 09:59:30 +0100 Subject: [PATCH 2/3] Rename input parameter to GenericDeleteFilter --- .../java/org/apache/iceberg/data/GenericDeleteFilter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/src/main/java/org/apache/iceberg/data/GenericDeleteFilter.java b/data/src/main/java/org/apache/iceberg/data/GenericDeleteFilter.java index e2b826639235..1faacb951bb8 100644 --- a/data/src/main/java/org/apache/iceberg/data/GenericDeleteFilter.java +++ b/data/src/main/java/org/apache/iceberg/data/GenericDeleteFilter.java @@ -49,12 +49,12 @@ public GenericDeleteFilter( } private GenericDeleteFilter( - Function inputFiles, + Function inputFilesProvider, FileScanTask task, Schema tableSchema, Schema requestedSchema) { super(task.file().location(), task.deletes(), tableSchema, requestedSchema); - this.inputFileProvider = inputFiles; + this.inputFileProvider = inputFileProvider; this.asStructLike = new InternalRecordWrapper(requiredSchema().asStruct()); } From b17bfc49a95ab1cf22356e3c015d804220290bd7 Mon Sep 17 00:00:00 2001 From: Gabor Kaszab Date: Tue, 17 Nov 2026 10:13:35 +0100 Subject: [PATCH 3/3] Fix compilation error --- .../main/java/org/apache/iceberg/data/GenericDeleteFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/src/main/java/org/apache/iceberg/data/GenericDeleteFilter.java b/data/src/main/java/org/apache/iceberg/data/GenericDeleteFilter.java index 1faacb951bb8..da5d513c7e66 100644 --- a/data/src/main/java/org/apache/iceberg/data/GenericDeleteFilter.java +++ b/data/src/main/java/org/apache/iceberg/data/GenericDeleteFilter.java @@ -49,7 +49,7 @@ public GenericDeleteFilter( } private GenericDeleteFilter( - Function inputFilesProvider, + Function inputFileProvider, FileScanTask task, Schema tableSchema, Schema requestedSchema) {