diff --git a/data/src/jmh/java/org/apache/iceberg/data/DeleteFilterBenchmark.java b/data/src/jmh/java/org/apache/iceberg/data/DeleteFilterBenchmark.java index ab918c9902d4..dbbfcab2548c 100644 --- a/data/src/jmh/java/org/apache/iceberg/data/DeleteFilterBenchmark.java +++ b/data/src/jmh/java/org/apache/iceberg/data/DeleteFilterBenchmark.java @@ -34,6 +34,8 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; +import org.apache.iceberg.encryption.EncryptingFileIO; +import org.apache.iceberg.encryption.InputFilesDecryptor; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; @@ -164,7 +166,10 @@ public void setupBenchmark() throws IOException { tasks = Lists.newArrayList(planned); } FileScanTask task = Iterables.getOnlyElement(tasks); - this.filter = new GenericDeleteFilter(table.io(), task, table.schema(), table.schema()); + InputFilesDecryptor decryptor = + InputFilesDecryptor.fromTasks( + tasks, EncryptingFileIO.combine(table.io(), table.encryption())); + this.filter = new GenericDeleteFilter(decryptor, task, table.schema(), table.schema()); this.filterInput = materializeFilterInput(rows, filter.requiredSchema()); } diff --git a/data/src/main/java/org/apache/iceberg/data/GenericDeleteFilter.java b/data/src/main/java/org/apache/iceberg/data/GenericDeleteFilter.java index 55fbe1aff34f..da5d513c7e66 100644 --- a/data/src/main/java/org/apache/iceberg/data/GenericDeleteFilter.java +++ b/data/src/main/java/org/apache/iceberg/data/GenericDeleteFilter.java @@ -22,6 +22,7 @@ import org.apache.iceberg.FileScanTask; import org.apache.iceberg.Schema; import org.apache.iceberg.StructLike; +import org.apache.iceberg.encryption.InputFilesDecryptor; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; @@ -29,12 +30,25 @@ public class GenericDeleteFilter extends DeleteFilter { private final Function inputFileProvider; private final InternalRecordWrapper asStructLike; + /** + * @deprecated since 1.13.0, will be removed in 1.14.0; use {@link + * #GenericDeleteFilter(InputFilesDecryptor, FileScanTask, Schema, Schema)} instead. + */ + @Deprecated public GenericDeleteFilter( FileIO io, FileScanTask task, Schema tableSchema, Schema requestedSchema) { this(io::newInputFile, task, tableSchema, requestedSchema); } - GenericDeleteFilter( + public GenericDeleteFilter( + InputFilesDecryptor decryptor, + FileScanTask task, + Schema tableSchema, + Schema requestedSchema) { + this(decryptor::getInputFile, task, tableSchema, requestedSchema); + } + + private GenericDeleteFilter( Function inputFileProvider, FileScanTask task, Schema tableSchema, diff --git a/data/src/main/java/org/apache/iceberg/data/GenericReader.java b/data/src/main/java/org/apache/iceberg/data/GenericReader.java index 12ea8d1bec8e..9c5afa4aa46f 100644 --- a/data/src/main/java/org/apache/iceberg/data/GenericReader.java +++ b/data/src/main/java/org/apache/iceberg/data/GenericReader.java @@ -62,7 +62,7 @@ public CloseableIterable open(CombinedScanTask task) { private CloseableIterable open(FileScanTask task, InputFilesDecryptor decryptor) { DeleteFilter deletes = - new GenericDeleteFilter(decryptor::getInputFile, task, tableSchema, projection); + new GenericDeleteFilter(decryptor, task, tableSchema, projection); Schema readSchema = deletes.requiredSchema(); CloseableIterable records = openFile(task, readSchema, decryptor); diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java index 764648e26302..88f8b1329a66 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java @@ -48,8 +48,8 @@ import org.apache.iceberg.data.IdentityPartitionConverters; import org.apache.iceberg.data.InternalRecordWrapper; import org.apache.iceberg.data.Record; -import org.apache.iceberg.encryption.EncryptedFiles; -import org.apache.iceberg.encryption.EncryptionManager; +import org.apache.iceberg.encryption.EncryptingFileIO; +import org.apache.iceberg.encryption.InputFilesDecryptor; import org.apache.iceberg.expressions.Evaluator; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; @@ -58,7 +58,6 @@ import org.apache.iceberg.hadoop.HadoopConfigurable; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.CloseableIterator; -import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; import org.apache.iceberg.mapping.NameMappingParser; import org.apache.iceberg.mr.Catalogs; @@ -236,8 +235,7 @@ private static final class IcebergRecordReader extends RecordReader private Iterator tasks; private T current; private CloseableIterator currentIterator; - private FileIO io; - private EncryptionManager encryptionManager; + private InputFilesDecryptor decryptor; @Override public void initialize(InputSplit split, TaskAttemptContext newContext) { @@ -248,8 +246,9 @@ public void initialize(InputSplit split, TaskAttemptContext newContext) { this.context = newContext; Table table = ((IcebergSplit) split).table(); checkAndSetIoConfig(conf, table); - this.io = table.io(); - this.encryptionManager = table.encryption(); + this.decryptor = + InputFilesDecryptor.fromTasks( + task.files(), EncryptingFileIO.combine(table.io(), table.encryption())); this.tasks = task.files().iterator(); this.tableSchema = InputFormatConfig.tableSchema(conf); this.nameMapping = table.properties().get(TableProperties.DEFAULT_NAME_MAPPING); @@ -311,9 +310,7 @@ public void close() throws IOException { @SuppressWarnings("unchecked") private CloseableIterable openTask(FileScanTask currentTask, Schema readSchema) { DataFile file = currentTask.file(); - InputFile inputFile = - encryptionManager.decrypt( - EncryptedFiles.encryptedInput(io.newInputFile(file.location()), file.keyMetadata())); + InputFile inputFile = decryptor.getInputFile(file.location()); ReadBuilder readBuilder = FormatModelRegistry.readBuilder(file.format(), Record.class, inputFile); @@ -343,7 +340,8 @@ private CloseableIterable openTask(FileScanTask currentTask, Schema readSchem @SuppressWarnings("unchecked") private CloseableIterable open(FileScanTask currentTask, Schema readSchema) { - DeleteFilter deletes = new GenericDeleteFilter(io, currentTask, tableSchema, readSchema); + DeleteFilter deletes = + new GenericDeleteFilter(decryptor, currentTask, tableSchema, readSchema); Schema requiredSchema = deletes.requiredSchema(); return deletes.filter(openTask(currentTask, requiredSchema)); } diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormatEncryption.java b/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormatEncryption.java new file mode 100644 index 000000000000..390d6846b6f5 --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormatEncryption.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.mr.mapreduce; + +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.apache.iceberg.types.Types.NestedField.required; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.TaskAttemptID; +import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; +import org.apache.iceberg.CombinedScanTask; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.Files; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SchemaParser; +import org.apache.iceberg.SerializableTable; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.Table; +import org.apache.iceberg.TestTables; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.encryption.EncryptingFileIO; +import org.apache.iceberg.encryption.EncryptionManager; +import org.apache.iceberg.encryption.EncryptionTestHelpers; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.mr.InputFormatConfig; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.Pair; +import org.apache.iceberg.util.StructLikeSet; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class TestIcebergInputFormatEncryption { + private static final Schema SCHEMA = + new Schema( + required(1, "id", Types.LongType.get()), optional(2, "data", Types.StringType.get())); + + @TempDir private File tableDir; + + private Table table; + + @BeforeEach + void createTable() { + EncryptionManager encryptionManager = EncryptionTestHelpers.createEncryptionManager(); + TestTables.TestTableOperations ops = + new TestTables.TestTableOperations( + "encrypted", + tableDir, + EncryptingFileIO.combine(new TestTables.LocalFileIO(), encryptionManager)) { + @Override + public EncryptionManager encryption() { + return encryptionManager; + } + }; + + this.table = + TestTables.create( + tableDir, + "encrypted", + SCHEMA, + PartitionSpec.unpartitioned(), + SortOrder.unsorted(), + 3, + ops); + } + + @AfterEach + void dropTable() { + TestTables.clearTables(); + } + + @Test + void readEncryptedDataFilesWithDV() throws IOException, InterruptedException { + Record first = record(1L, "a"); + Record second = record(2L, "b"); + Record third = record(3L, "c"); + DataFile firstFile = + FileHelpers.writeDataFile(table, outputFile("first.parquet"), List.of(first, second)); + DataFile secondFile = + FileHelpers.writeDataFile(table, outputFile("second.parquet"), List.of(third)); + table.newAppend().appendFile(firstFile).appendFile(secondFile).commit(); + + DeleteFile dv = + FileHelpers.writeDeleteFile(table, null, List.of(Pair.of(firstFile.location(), 0L)), 3) + .first(); + table.newRowDelta().addDeletes(dv).commit(); + + assertThat(firstFile.keyMetadata()).isNotNull(); + assertThat(secondFile.keyMetadata()).isNotNull(); + assertThat(dv.keyMetadata()).isNotNull(); + + StructLikeSet expected = StructLikeSet.create(SCHEMA.asStruct()); + expected.add(second); + expected.add(third); + + assertThat(readWithInputFormat()).isEqualTo(expected); + } + + private StructLikeSet readWithInputFormat() throws IOException, InterruptedException { + Configuration conf = new Configuration(); + conf.set(InputFormatConfig.TABLE_SCHEMA, SchemaParser.toJson(table.schema())); + TaskAttemptContext context = new TaskAttemptContextImpl(conf, new TaskAttemptID()); + IcebergInputFormat inputFormat = new IcebergInputFormat<>(); + Table serializableTable = SerializableTable.copyOf(table); + + StructLikeSet actual = StructLikeSet.create(SCHEMA.asStruct()); + try (CloseableIterable tasks = table.newScan().planTasks()) { + for (CombinedScanTask task : tasks) { + IcebergSplit split = new IcebergSplit(serializableTable, conf, task); + try (RecordReader reader = inputFormat.createRecordReader(split, context)) { + reader.initialize(split, context); + while (reader.nextKeyValue()) { + actual.add(reader.getCurrentValue()); + } + } + } + } + + return actual; + } + + private static Record record(long id, String data) { + Record record = GenericRecord.create(SCHEMA); + record.setField("id", id); + record.setField("data", data); + return record; + } + + private OutputFile outputFile(String fileName) { + return Files.localOutput(new File(tableDir, fileName)); + } +}