Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@
import org.apache.iceberg.Schema;
import org.apache.iceberg.Table;
import org.apache.iceberg.TableProperties;
import org.apache.iceberg.encryption.EncryptingFileIO;
import org.apache.iceberg.encryption.InputFilesDecryptor;
import org.apache.iceberg.hadoop.HadoopTables;
import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
Expand Down Expand Up @@ -124,7 +126,7 @@
table, Files.localOutput(new File(tableDir, "data.parquet")), rows);
table.newAppend().appendFile(dataFile).commit();

switch (deleteType) {

Check warning on line 129 in data/src/jmh/java/org/apache/iceberg/data/DeleteFilterBenchmark.java

View workflow job for this annotation

GitHub Actions / build-checks (17, pull_request)

[StatementSwitchToExpressionSwitch] This statement switch can be converted to a new-style arrow switch
case "POSITION":
List<Pair<CharSequence, Long>> posDeletes = Lists.newArrayList();
for (long pos = 0; pos < NUM_ROWS; pos += DELETE_EVERY) {
Expand Down Expand Up @@ -164,7 +166,10 @@
tasks = Lists.newArrayList(planned);
}
FileScanTask task = Iterables.getOnlyElement(tasks);
this.filter = new GenericDeleteFilter(table.io(), task, table.schema(), table.schema());
InputFilesDecryptor decryptor =
InputFilesDecryptor.fromTasks(
tasks, EncryptingFileIO.combine(table.io(), table.encryption()));
this.filter = new GenericDeleteFilter(decryptor, task, table.schema(), table.schema());
this.filterInput = materializeFilterInput(rows, filter.requiredSchema());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,19 +22,33 @@
import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.Schema;
import org.apache.iceberg.StructLike;
import org.apache.iceberg.encryption.InputFilesDecryptor;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.io.InputFile;

public class GenericDeleteFilter extends DeleteFilter<Record> {
private final Function<String, InputFile> inputFileProvider;
private final InternalRecordWrapper asStructLike;

/**
* @deprecated since 1.13.0, will be removed in 1.14.0; use {@link
* #GenericDeleteFilter(InputFilesDecryptor, FileScanTask, Schema, Schema)} instead.
*/
@Deprecated
public GenericDeleteFilter(
FileIO io, FileScanTask task, Schema tableSchema, Schema requestedSchema) {
this(io::newInputFile, task, tableSchema, requestedSchema);
}

GenericDeleteFilter(
public GenericDeleteFilter(
InputFilesDecryptor decryptor,
FileScanTask task,
Schema tableSchema,
Schema requestedSchema) {
this(decryptor::getInputFile, task, tableSchema, requestedSchema);
}

private GenericDeleteFilter(
Function<String, InputFile> inputFileProvider,
FileScanTask task,
Schema tableSchema,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ public CloseableIterable<Record> open(CombinedScanTask task) {

private CloseableIterable<Record> open(FileScanTask task, InputFilesDecryptor decryptor) {
DeleteFilter<Record> deletes =
new GenericDeleteFilter(decryptor::getInputFile, task, tableSchema, projection);
new GenericDeleteFilter(decryptor, task, tableSchema, projection);
Schema readSchema = deletes.requiredSchema();

CloseableIterable<Record> records = openFile(task, readSchema, decryptor);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@
import org.apache.iceberg.data.IdentityPartitionConverters;
import org.apache.iceberg.data.InternalRecordWrapper;
import org.apache.iceberg.data.Record;
import org.apache.iceberg.encryption.EncryptedFiles;
import org.apache.iceberg.encryption.EncryptionManager;
import org.apache.iceberg.encryption.EncryptingFileIO;
import org.apache.iceberg.encryption.InputFilesDecryptor;
import org.apache.iceberg.expressions.Evaluator;
import org.apache.iceberg.expressions.Expression;
import org.apache.iceberg.expressions.Expressions;
Expand All @@ -58,7 +58,6 @@
import org.apache.iceberg.hadoop.HadoopConfigurable;
import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.io.CloseableIterator;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.io.InputFile;
import org.apache.iceberg.mapping.NameMappingParser;
import org.apache.iceberg.mr.Catalogs;
Expand Down Expand Up @@ -236,8 +235,7 @@ private static final class IcebergRecordReader<T> extends RecordReader<Void, T>
private Iterator<FileScanTask> tasks;
private T current;
private CloseableIterator<T> currentIterator;
private FileIO io;
private EncryptionManager encryptionManager;
private InputFilesDecryptor decryptor;

@Override
public void initialize(InputSplit split, TaskAttemptContext newContext) {
Expand All @@ -248,8 +246,9 @@ public void initialize(InputSplit split, TaskAttemptContext newContext) {
this.context = newContext;
Table table = ((IcebergSplit) split).table();
checkAndSetIoConfig(conf, table);
this.io = table.io();
this.encryptionManager = table.encryption();
this.decryptor =
InputFilesDecryptor.fromTasks(
task.files(), EncryptingFileIO.combine(table.io(), table.encryption()));
this.tasks = task.files().iterator();
this.tableSchema = InputFormatConfig.tableSchema(conf);
this.nameMapping = table.properties().get(TableProperties.DEFAULT_NAME_MAPPING);
Expand Down Expand Up @@ -311,9 +310,7 @@ public void close() throws IOException {
@SuppressWarnings("unchecked")
private CloseableIterable<T> openTask(FileScanTask currentTask, Schema readSchema) {
DataFile file = currentTask.file();
InputFile inputFile =
encryptionManager.decrypt(
EncryptedFiles.encryptedInput(io.newInputFile(file.location()), file.keyMetadata()));
InputFile inputFile = decryptor.getInputFile(file.location());

ReadBuilder<Record, ?> readBuilder =
FormatModelRegistry.readBuilder(file.format(), Record.class, inputFile);
Expand Down Expand Up @@ -343,7 +340,8 @@ private CloseableIterable<T> openTask(FileScanTask currentTask, Schema readSchem

@SuppressWarnings("unchecked")
private CloseableIterable<T> open(FileScanTask currentTask, Schema readSchema) {
DeleteFilter deletes = new GenericDeleteFilter(io, currentTask, tableSchema, readSchema);
DeleteFilter deletes =
new GenericDeleteFilter(decryptor, currentTask, tableSchema, readSchema);
Schema requiredSchema = deletes.requiredSchema();
return deletes.filter(openTask(currentTask, requiredSchema));
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.iceberg.mr.mapreduce;

import static org.apache.iceberg.types.Types.NestedField.optional;
import static org.apache.iceberg.types.Types.NestedField.required;
import static org.assertj.core.api.Assertions.assertThat;

import java.io.File;
import java.io.IOException;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
import org.apache.iceberg.CombinedScanTask;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.DeleteFile;
import org.apache.iceberg.Files;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Schema;
import org.apache.iceberg.SchemaParser;
import org.apache.iceberg.SerializableTable;
import org.apache.iceberg.SortOrder;
import org.apache.iceberg.Table;
import org.apache.iceberg.TestTables;
import org.apache.iceberg.data.FileHelpers;
import org.apache.iceberg.data.GenericRecord;
import org.apache.iceberg.data.Record;
import org.apache.iceberg.encryption.EncryptingFileIO;
import org.apache.iceberg.encryption.EncryptionManager;
import org.apache.iceberg.encryption.EncryptionTestHelpers;
import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.io.OutputFile;
import org.apache.iceberg.mr.InputFormatConfig;
import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.Pair;
import org.apache.iceberg.util.StructLikeSet;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

class TestIcebergInputFormatEncryption {
private static final Schema SCHEMA =
new Schema(
required(1, "id", Types.LongType.get()), optional(2, "data", Types.StringType.get()));

@TempDir private File tableDir;

private Table table;

@BeforeEach
void createTable() {
EncryptionManager encryptionManager = EncryptionTestHelpers.createEncryptionManager();
TestTables.TestTableOperations ops =
new TestTables.TestTableOperations(
"encrypted",
tableDir,
EncryptingFileIO.combine(new TestTables.LocalFileIO(), encryptionManager)) {
@Override
public EncryptionManager encryption() {
return encryptionManager;
}
};

this.table =
TestTables.create(
tableDir,
"encrypted",
SCHEMA,
PartitionSpec.unpartitioned(),
SortOrder.unsorted(),
3,
ops);
}

@AfterEach
void dropTable() {
TestTables.clearTables();
}

@Test
void readEncryptedDataFilesWithDV() throws IOException, InterruptedException {
Record first = record(1L, "a");
Record second = record(2L, "b");
Record third = record(3L, "c");
DataFile firstFile =
FileHelpers.writeDataFile(table, outputFile("first.parquet"), List.of(first, second));
DataFile secondFile =
FileHelpers.writeDataFile(table, outputFile("second.parquet"), List.of(third));
table.newAppend().appendFile(firstFile).appendFile(secondFile).commit();

DeleteFile dv =
FileHelpers.writeDeleteFile(table, null, List.of(Pair.of(firstFile.location(), 0L)), 3)
.first();
table.newRowDelta().addDeletes(dv).commit();

assertThat(firstFile.keyMetadata()).isNotNull();

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same question

assertThat(secondFile.keyMetadata()).isNotNull();
assertThat(dv.keyMetadata()).isNotNull();

StructLikeSet expected = StructLikeSet.create(SCHEMA.asStruct());
expected.add(second);
expected.add(third);

assertThat(readWithInputFormat()).isEqualTo(expected);
}

private StructLikeSet readWithInputFormat() throws IOException, InterruptedException {
Configuration conf = new Configuration();
conf.set(InputFormatConfig.TABLE_SCHEMA, SchemaParser.toJson(table.schema()));
TaskAttemptContext context = new TaskAttemptContextImpl(conf, new TaskAttemptID());
IcebergInputFormat<Record> inputFormat = new IcebergInputFormat<>();
Table serializableTable = SerializableTable.copyOf(table);

StructLikeSet actual = StructLikeSet.create(SCHEMA.asStruct());
try (CloseableIterable<CombinedScanTask> tasks = table.newScan().planTasks()) {
for (CombinedScanTask task : tasks) {
IcebergSplit split = new IcebergSplit(serializableTable, conf, task);
try (RecordReader<Void, Record> reader = inputFormat.createRecordReader(split, context)) {
reader.initialize(split, context);
while (reader.nextKeyValue()) {
actual.add(reader.getCurrentValue());
}
}
}
}

return actual;
}

private static Record record(long id, String data) {
Record record = GenericRecord.create(SCHEMA);
record.setField("id", id);
record.setField("data", data);
return record;
}

private OutputFile outputFile(String fileName) {
return Files.localOutput(new File(tableDir, fileName));
}
}
Loading