Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 2 additions & 9 deletions api/src/main/java/org/apache/iceberg/ContentFile.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
*
* @param <F> the concrete Java class of a ContentFile instance.
*/
public interface ContentFile<F> {
public interface ContentFile<F> extends EncryptedFile {
/**
* Returns the path of the manifest which this file is referenced in or null if it was not read
* from a manifest.
Expand Down Expand Up @@ -60,6 +60,7 @@ default String manifestLocation() {
CharSequence path();

/** Return the fully qualified path to the file. */
@Override
default String location() {
return path().toString();
}
Expand All @@ -73,9 +74,6 @@ default String location() {
/** Returns the number of top-level records in the file. */
long recordCount();

/** Returns the file size in bytes. */
long fileSizeInBytes();

/**
* Returns if collected, map from column ID to the size of the column in bytes, null otherwise.
*/
Expand Down Expand Up @@ -129,11 +127,6 @@ default Map<Integer, Integer> avgValueSizes() {
return Metrics.avgValueSizes(totalBytes(), valueCounts(), nullValueCounts());
}

/**
* Returns metadata about how this file is encrypted, or null if the file is stored in plain text.
*/
ByteBuffer keyMetadata();

/**
* Returns list of recommended split locations, if applicable, null otherwise.
*
Expand Down
36 changes: 36 additions & 0 deletions api/src/main/java/org/apache/iceberg/EncryptedFile.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.iceberg;

import java.nio.ByteBuffer;

/** A file that may be encrypted. */
public interface EncryptedFile {

/** Returns the location of the file. */
String location();

/** Returns the size of the file in bytes. */
long fileSizeInBytes();

/**
* Returns metadata about how this file is encrypted, or null if the file is stored in plain text.
*/
ByteBuffer keyMetadata();
}
13 changes: 12 additions & 1 deletion api/src/main/java/org/apache/iceberg/ManifestFile.java
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
import org.apache.iceberg.types.Types;

/** Represents a manifest file that can be scanned to find files in a table. */
public interface ManifestFile {
public interface ManifestFile extends EncryptedFile {
int PARTITION_SUMMARIES_ELEMENT_ID = 508;

Types.NestedField PATH =
Expand Down Expand Up @@ -123,9 +123,19 @@ static Schema schema() {
/** Returns fully qualified path to the file, suitable for constructing a Hadoop Path. */
String path();

@gaborkaszab gaborkaszab Nov 17, 2026 •

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should deprecate path() and replace usage to location(), similarly to how we have these in ContentFile already.
Same for length() and fileSizeInBytes()


@Override
default String location() {
return path();
}

/** Returns length of the manifest file. */
long length();

@Override
default long fileSizeInBytes() {
return length();
}

/** Returns ID of the {@link PartitionSpec} used to write the manifest file. */
int partitionSpecId();

Expand Down Expand Up @@ -221,6 +231,7 @@ default Long modifiedRowsCount() {
* Returns metadata about how this manifest file is encrypted, or null if the file is stored in
* plain text.
*/
@Override
default ByteBuffer keyMetadata() {
return null;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,7 @@
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
import org.apache.iceberg.ContentFile;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.DeleteFile;
import org.apache.iceberg.ManifestFile;
import org.apache.iceberg.EncryptedFile;
import org.apache.iceberg.ManifestListFile;
import org.apache.iceberg.io.BulkDeletionFailureException;
import org.apache.iceberg.io.DelegateFileIO;
Expand Down Expand Up @@ -68,11 +65,11 @@ public static EncryptingFileIO combine(FileIO io, EncryptionManager em) {
this.em = em;
}

public Map<String, InputFile> bulkDecrypt(Iterable<? extends ContentFile<?>> files) {
public Map<String, InputFile> bulkDecrypt(Iterable<? extends EncryptedFile> files) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This requires changes in caller code. Are we comfortable with this?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We are widening the parameter type, so revapi doesn't complain. I'm not entirely sure if this counts as breaking change. Internally, once #18486 is merged, the Spark usage is gone, and the only non-test usage will be from InputFilesDecryptor that is the end goal.

For custom implementations: the constructor of EncryptingFileIO is package private, so custom implementations aren't feasible outside the package. I'd say this is safe then.

ImmutableMap.Builder<String, InputFile> builder = ImmutableMap.builder();

List<EncryptedInputFile> encryptedFiles = Lists.newArrayList();
for (ContentFile<?> file : files) {
for (EncryptedFile file : files) {
if (file.keyMetadata() != null) {
encryptedFiles.add(wrap(file));
} else {
Expand Down Expand Up @@ -104,42 +101,24 @@ public InputFile newInputFile(String path, long length) {
}

@Override
public InputFile newInputFile(DataFile file) {
return newInputFile((ContentFile<?>) file);
public InputFile newInputFile(ManifestListFile manifestList) {
if (manifestList.encryptionKeyID() != null) {
ByteBuffer keyMetadata = manifestList.decryptKeyMetadata(em);
return newDecryptingInputFile(manifestList.location(), keyMetadata);
} else {
return newInputFile(manifestList.location());
}
}

@Override
public InputFile newInputFile(DeleteFile file) {
return newInputFile((ContentFile<?>) file);
}

private InputFile newInputFile(ContentFile<?> file) {
public InputFile newInputFile(EncryptedFile file) {
if (file.keyMetadata() != null) {
return newDecryptingInputFile(file.location(), file.fileSizeInBytes(), file.keyMetadata());
} else {
return newInputFile(file.location(), file.fileSizeInBytes());
}
}

@Override
public InputFile newInputFile(ManifestFile manifest) {
if (manifest.keyMetadata() != null) {
return newDecryptingInputFile(manifest.path(), manifest.length(), manifest.keyMetadata());
} else {
return newInputFile(manifest.path(), manifest.length());
}
}

@Override
public InputFile newInputFile(ManifestListFile manifestList) {
if (manifestList.encryptionKeyID() != null) {
ByteBuffer keyMetadata = manifestList.decryptKeyMetadata(em);
return newDecryptingInputFile(manifestList.location(), keyMetadata);
} else {
return newInputFile(manifestList.location());
}
}

public InputFile newDecryptingInputFile(String path, ByteBuffer buffer) {
return em.decrypt(wrap(io.newInputFile(path), buffer));
}
Expand Down Expand Up @@ -181,7 +160,7 @@ public void close() {
}
}

private SimpleEncryptedInputFile wrap(ContentFile<?> file) {
private SimpleEncryptedInputFile wrap(EncryptedFile file) {
InputFile encryptedInputFile = io.newInputFile(file.location(), file.fileSizeInBytes());
return new SimpleEncryptedInputFile(encryptedInputFile, toKeyMetadata(file.keyMetadata()));
}
Expand Down
27 changes: 12 additions & 15 deletions api/src/main/java/org/apache/iceberg/io/FileIO.java
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.util.Map;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.DeleteFile;
import org.apache.iceberg.EncryptedFile;
import org.apache.iceberg.ManifestFile;
import org.apache.iceberg.ManifestListFile;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
Expand All @@ -48,27 +49,15 @@ default InputFile newInputFile(String path, long length) {
}

default InputFile newInputFile(DataFile file) {
Preconditions.checkArgument(
file.keyMetadata() == null,
"Cannot decrypt data file: %s (use EncryptingFileIO)",
file.location());
return newInputFile(file.location(), file.fileSizeInBytes());
return newInputFile((EncryptedFile) file);
}

default InputFile newInputFile(DeleteFile file) {
Preconditions.checkArgument(
file.keyMetadata() == null,
"Cannot decrypt delete file: %s (use EncryptingFileIO)",
file.location());
return newInputFile(file.location(), file.fileSizeInBytes());
return newInputFile((EncryptedFile) file);
}

default InputFile newInputFile(ManifestFile manifest) {
Preconditions.checkArgument(
manifest.keyMetadata() == null,
"Cannot decrypt manifest: %s (use EncryptingFileIO)",
manifest.path());
return newInputFile(manifest.path(), manifest.length());
return newInputFile((EncryptedFile) manifest);
}

default InputFile newInputFile(ManifestListFile manifestList) {

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I dropped BaseManifestListFile because that was package private. Now, there is no implementation in the library that can call this function, still I don't think we can drop this, because it'd break API for users that happen to implement their own ManifestListFile. Not likely, but technically feasible.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Rather than deprecating each of these, we could update all of the exiting *File types to implement EncryptedFile which would then have a common way to reference the necessary fields without having to update all of the places where they are used with FileIO.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For me it seems like a good compromise, to leave out ManifestListFile but standardize on the other files.
What do you think @danielcweeks?

Expand All @@ -80,6 +69,14 @@ default InputFile newInputFile(ManifestListFile manifestList) {
return newInputFile(manifestList.location());
}

default InputFile newInputFile(EncryptedFile file) {
Preconditions.checkArgument(
file.keyMetadata() == null,
"Cannot decrypt file: %s (use EncryptingFileIO)",
file.location());
return newInputFile(file.location(), file.fileSizeInBytes());
}

/** Get a {@link OutputFile} instance to write bytes to the file at the given path. */
OutputFile newOutputFile(String path);

Expand Down
12 changes: 1 addition & 11 deletions core/src/main/java/org/apache/iceberg/ColumnFile.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,10 @@
*/
package org.apache.iceberg;

import java.nio.ByteBuffer;
import java.util.Set;
import org.apache.iceberg.types.Types;

interface ColumnFile {
interface ColumnFile extends EncryptedFile {
Types.NestedField LOCATION =
Types.NestedField.required(
160, "location", Types.StringType.get(), "Location of the column file");
Expand Down Expand Up @@ -52,21 +51,12 @@ static Types.StructType schema() {
return Types.StructType.of(LOCATION, FIELD_IDS, FILE_FORMAT, FILE_SIZE_IN_BYTES, KEY_METADATA);
}

/** Returns the location of this column file. */
String location();

/** Returns the field IDs contained in this column file. */
Set<Integer> fieldIds();

/** Returns the format of this column file. */
FileFormat fileFormat();

/** Returns the total size of this column file in bytes. */
long fileSizeInBytes();

/** Returns encryption key metadata, or null if this column file is not encrypted. */
ByteBuffer keyMetadata();

/** Copies this column file. */
ColumnFile copy();
}
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ class TestV4ManifestReader {
static {
Mockito.when(UNREAD_MANIFEST_FILE.path())
.thenReturn(FileFormat.PARQUET.addExtension("manifest"));
Mockito.when(UNREAD_MANIFEST_FILE.location())
.thenReturn(FileFormat.PARQUET.addExtension("manifest"));
Mockito.when(UNREAD_MANIFEST_FILE.formatVersion()).thenReturn(4);
Mockito.when(UNREAD_MANIFEST_FILE.content()).thenReturn(ManifestContent.DATA);
}
Expand Down Expand Up @@ -1732,6 +1734,7 @@ public void partitionFilterKeepsUnpartitionedFile(FileFormat format) throws IOEx
public void validationUnknownManifestFormatFailure() {
ManifestFile fileWithoutFormat = Mockito.mock(ManifestFile.class);
Mockito.when(fileWithoutFormat.path()).thenReturn("manifest");
Mockito.when(fileWithoutFormat.location()).thenReturn("manifest");
Mockito.when(fileWithoutFormat.formatVersion()).thenReturn(4);
Mockito.when(fileWithoutFormat.content()).thenReturn(ManifestContent.DATA);

Expand All @@ -1749,6 +1752,7 @@ public void validationUnknownManifestFormatFailure() {
public void validationUnknownFormatVersionFailure() {
ManifestFile v3File = Mockito.mock(ManifestFile.class);
Mockito.when(v3File.path()).thenReturn(FileFormat.PARQUET.addExtension("manifest"));
Mockito.when(v3File.location()).thenReturn(FileFormat.PARQUET.addExtension("manifest"));
Mockito.when(v3File.formatVersion()).thenReturn(3);
Mockito.when(v3File.content()).thenReturn(ManifestContent.DATA);

Expand All @@ -1762,6 +1766,7 @@ public void validationUnknownFormatVersionFailure() {
public void validationDeleteManifestFailure() {
ManifestFile v3File = Mockito.mock(ManifestFile.class);
Mockito.when(v3File.path()).thenReturn(FileFormat.PARQUET.addExtension("manifest"));
Mockito.when(v3File.location()).thenReturn(FileFormat.PARQUET.addExtension("manifest"));
Mockito.when(v3File.formatVersion()).thenReturn(4);
Mockito.when(v3File.content()).thenReturn(ManifestContent.DELETES);

Expand Down Expand Up @@ -2118,6 +2123,7 @@ private ManifestFile writeManifest(
private static ManifestFile v4Manifest(String path) {
ManifestFile manifest = mock(ManifestFile.class);
when(manifest.path()).thenReturn(path);
when(manifest.location()).thenReturn(path);
when(manifest.formatVersion()).thenReturn(FORMAT_VERSION_V4);
when(manifest.content()).thenReturn(ManifestContent.DATA);
when(manifest.snapshotId()).thenReturn(SNAPSHOT_ID);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.apache.iceberg.ContentFile;
import org.apache.iceberg.ContentScanTask;
import org.apache.iceberg.DeleteFile;
import org.apache.iceberg.EncryptedFile;
import org.apache.iceberg.MetadataColumns;
import org.apache.iceberg.Partitioning;
import org.apache.iceberg.ScanTask;
Expand Down Expand Up @@ -185,7 +186,10 @@ private Map<String, InputFile> inputFiles() {
if (lazyInputFiles == null) {
this.lazyInputFiles =
fileIO.bulkDecrypt(
() -> taskGroup.tasks().stream().flatMap(this::referencedFiles).iterator());
() ->
taskGroup.tasks().stream()
.<EncryptedFile>flatMap(this::referencedFiles)
.iterator());
}

return lazyInputFiles;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.apache.iceberg.ContentFile;
import org.apache.iceberg.ContentScanTask;
import org.apache.iceberg.DeleteFile;
import org.apache.iceberg.EncryptedFile;
import org.apache.iceberg.MetadataColumns;
import org.apache.iceberg.Partitioning;
import org.apache.iceberg.ScanTask;
Expand Down Expand Up @@ -185,7 +186,10 @@ private Map<String, InputFile> inputFiles() {
if (lazyInputFiles == null) {
this.lazyInputFiles =
fileIO.bulkDecrypt(
() -> taskGroup.tasks().stream().flatMap(this::referencedFiles).iterator());
() ->

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These changes will be gone with #18486

taskGroup.tasks().stream()
.<EncryptedFile>flatMap(this::referencedFiles)
.iterator());
}

return lazyInputFiles;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.apache.iceberg.ContentFile;
import org.apache.iceberg.ContentScanTask;
import org.apache.iceberg.DeleteFile;
import org.apache.iceberg.EncryptedFile;
import org.apache.iceberg.MetadataColumns;
import org.apache.iceberg.Partitioning;
import org.apache.iceberg.ScanTask;
Expand Down Expand Up @@ -194,7 +195,10 @@ private Map<String, InputFile> inputFiles() {
if (lazyInputFiles == null) {
this.lazyInputFiles =
fileIO.bulkDecrypt(
() -> taskGroup.tasks().stream().flatMap(this::referencedFiles).iterator());
() ->
taskGroup.tasks().stream()
.<EncryptedFile>flatMap(this::referencedFiles)
.iterator());
}

return lazyInputFiles;
Expand Down
Loading
Loading