diff --git a/api/src/main/java/org/apache/iceberg/ContentFile.java b/api/src/main/java/org/apache/iceberg/ContentFile.java index c6dff4a934a7..0e93cb27eefb 100644 --- a/api/src/main/java/org/apache/iceberg/ContentFile.java +++ b/api/src/main/java/org/apache/iceberg/ContentFile.java @@ -28,7 +28,7 @@ * * @param the concrete Java class of a ContentFile instance. */ -public interface ContentFile { +public interface ContentFile extends EncryptedFile { /** * Returns the path of the manifest which this file is referenced in or null if it was not read * from a manifest. @@ -60,6 +60,7 @@ default String manifestLocation() { CharSequence path(); /** Return the fully qualified path to the file. */ + @Override default String location() { return path().toString(); } @@ -73,9 +74,6 @@ default String location() { /** Returns the number of top-level records in the file. */ long recordCount(); - /** Returns the file size in bytes. */ - long fileSizeInBytes(); - /** * Returns if collected, map from column ID to the size of the column in bytes, null otherwise. */ @@ -129,11 +127,6 @@ default Map avgValueSizes() { return Metrics.avgValueSizes(totalBytes(), valueCounts(), nullValueCounts()); } - /** - * Returns metadata about how this file is encrypted, or null if the file is stored in plain text. - */ - ByteBuffer keyMetadata(); - /** * Returns list of recommended split locations, if applicable, null otherwise. * diff --git a/api/src/main/java/org/apache/iceberg/EncryptedFile.java b/api/src/main/java/org/apache/iceberg/EncryptedFile.java new file mode 100644 index 000000000000..f2395de6bbc7 --- /dev/null +++ b/api/src/main/java/org/apache/iceberg/EncryptedFile.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.nio.ByteBuffer; + +/** A file that may be encrypted. */ +public interface EncryptedFile { + + /** Returns the location of the file. */ + String location(); + + /** Returns the size of the file in bytes. */ + long fileSizeInBytes(); + + /** + * Returns metadata about how this file is encrypted, or null if the file is stored in plain text. + */ + ByteBuffer keyMetadata(); +} diff --git a/api/src/main/java/org/apache/iceberg/ManifestFile.java b/api/src/main/java/org/apache/iceberg/ManifestFile.java index 32c7adae679f..733afca7471b 100644 --- a/api/src/main/java/org/apache/iceberg/ManifestFile.java +++ b/api/src/main/java/org/apache/iceberg/ManifestFile.java @@ -26,7 +26,7 @@ import org.apache.iceberg.types.Types; /** Represents a manifest file that can be scanned to find files in a table. */ -public interface ManifestFile { +public interface ManifestFile extends EncryptedFile { int PARTITION_SUMMARIES_ELEMENT_ID = 508; Types.NestedField PATH = @@ -123,9 +123,19 @@ static Schema schema() { /** Returns fully qualified path to the file, suitable for constructing a Hadoop Path. */ String path(); + @Override + default String location() { + return path(); + } + /** Returns length of the manifest file. */ long length(); + @Override + default long fileSizeInBytes() { + return length(); + } + /** Returns ID of the {@link PartitionSpec} used to write the manifest file. */ int partitionSpecId(); @@ -221,6 +231,7 @@ default Long modifiedRowsCount() { * Returns metadata about how this manifest file is encrypted, or null if the file is stored in * plain text. */ + @Override default ByteBuffer keyMetadata() { return null; } diff --git a/api/src/main/java/org/apache/iceberg/encryption/EncryptingFileIO.java b/api/src/main/java/org/apache/iceberg/encryption/EncryptingFileIO.java index 6a14db3dd439..be2491d8f368 100644 --- a/api/src/main/java/org/apache/iceberg/encryption/EncryptingFileIO.java +++ b/api/src/main/java/org/apache/iceberg/encryption/EncryptingFileIO.java @@ -25,10 +25,7 @@ import java.nio.ByteBuffer; import java.util.List; import java.util.Map; -import org.apache.iceberg.ContentFile; -import org.apache.iceberg.DataFile; -import org.apache.iceberg.DeleteFile; -import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.EncryptedFile; import org.apache.iceberg.ManifestListFile; import org.apache.iceberg.io.BulkDeletionFailureException; import org.apache.iceberg.io.DelegateFileIO; @@ -68,11 +65,11 @@ public static EncryptingFileIO combine(FileIO io, EncryptionManager em) { this.em = em; } - public Map bulkDecrypt(Iterable> files) { + public Map bulkDecrypt(Iterable files) { ImmutableMap.Builder builder = ImmutableMap.builder(); List encryptedFiles = Lists.newArrayList(); - for (ContentFile file : files) { + for (EncryptedFile file : files) { if (file.keyMetadata() != null) { encryptedFiles.add(wrap(file)); } else { @@ -104,16 +101,17 @@ public InputFile newInputFile(String path, long length) { } @Override - public InputFile newInputFile(DataFile file) { - return newInputFile((ContentFile) file); + public InputFile newInputFile(ManifestListFile manifestList) { + if (manifestList.encryptionKeyID() != null) { + ByteBuffer keyMetadata = manifestList.decryptKeyMetadata(em); + return newDecryptingInputFile(manifestList.location(), keyMetadata); + } else { + return newInputFile(manifestList.location()); + } } @Override - public InputFile newInputFile(DeleteFile file) { - return newInputFile((ContentFile) file); - } - - private InputFile newInputFile(ContentFile file) { + public InputFile newInputFile(EncryptedFile file) { if (file.keyMetadata() != null) { return newDecryptingInputFile(file.location(), file.fileSizeInBytes(), file.keyMetadata()); } else { @@ -121,25 +119,6 @@ private InputFile newInputFile(ContentFile file) { } } - @Override - public InputFile newInputFile(ManifestFile manifest) { - if (manifest.keyMetadata() != null) { - return newDecryptingInputFile(manifest.path(), manifest.length(), manifest.keyMetadata()); - } else { - return newInputFile(manifest.path(), manifest.length()); - } - } - - @Override - public InputFile newInputFile(ManifestListFile manifestList) { - if (manifestList.encryptionKeyID() != null) { - ByteBuffer keyMetadata = manifestList.decryptKeyMetadata(em); - return newDecryptingInputFile(manifestList.location(), keyMetadata); - } else { - return newInputFile(manifestList.location()); - } - } - public InputFile newDecryptingInputFile(String path, ByteBuffer buffer) { return em.decrypt(wrap(io.newInputFile(path), buffer)); } @@ -181,7 +160,7 @@ public void close() { } } - private SimpleEncryptedInputFile wrap(ContentFile file) { + private SimpleEncryptedInputFile wrap(EncryptedFile file) { InputFile encryptedInputFile = io.newInputFile(file.location(), file.fileSizeInBytes()); return new SimpleEncryptedInputFile(encryptedInputFile, toKeyMetadata(file.keyMetadata())); } diff --git a/api/src/main/java/org/apache/iceberg/io/FileIO.java b/api/src/main/java/org/apache/iceberg/io/FileIO.java index 72d2d9b0fca2..5ea4fe432d76 100644 --- a/api/src/main/java/org/apache/iceberg/io/FileIO.java +++ b/api/src/main/java/org/apache/iceberg/io/FileIO.java @@ -23,6 +23,7 @@ import java.util.Map; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.EncryptedFile; import org.apache.iceberg.ManifestFile; import org.apache.iceberg.ManifestListFile; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; @@ -48,27 +49,15 @@ default InputFile newInputFile(String path, long length) { } default InputFile newInputFile(DataFile file) { - Preconditions.checkArgument( - file.keyMetadata() == null, - "Cannot decrypt data file: %s (use EncryptingFileIO)", - file.location()); - return newInputFile(file.location(), file.fileSizeInBytes()); + return newInputFile((EncryptedFile) file); } default InputFile newInputFile(DeleteFile file) { - Preconditions.checkArgument( - file.keyMetadata() == null, - "Cannot decrypt delete file: %s (use EncryptingFileIO)", - file.location()); - return newInputFile(file.location(), file.fileSizeInBytes()); + return newInputFile((EncryptedFile) file); } default InputFile newInputFile(ManifestFile manifest) { - Preconditions.checkArgument( - manifest.keyMetadata() == null, - "Cannot decrypt manifest: %s (use EncryptingFileIO)", - manifest.path()); - return newInputFile(manifest.path(), manifest.length()); + return newInputFile((EncryptedFile) manifest); } default InputFile newInputFile(ManifestListFile manifestList) { @@ -80,6 +69,14 @@ default InputFile newInputFile(ManifestListFile manifestList) { return newInputFile(manifestList.location()); } + default InputFile newInputFile(EncryptedFile file) { + Preconditions.checkArgument( + file.keyMetadata() == null, + "Cannot decrypt file: %s (use EncryptingFileIO)", + file.location()); + return newInputFile(file.location(), file.fileSizeInBytes()); + } + /** Get a {@link OutputFile} instance to write bytes to the file at the given path. */ OutputFile newOutputFile(String path); diff --git a/core/src/main/java/org/apache/iceberg/ColumnFile.java b/core/src/main/java/org/apache/iceberg/ColumnFile.java index 7bdab8d88dd2..3ea203d71201 100644 --- a/core/src/main/java/org/apache/iceberg/ColumnFile.java +++ b/core/src/main/java/org/apache/iceberg/ColumnFile.java @@ -18,11 +18,10 @@ */ package org.apache.iceberg; -import java.nio.ByteBuffer; import java.util.Set; import org.apache.iceberg.types.Types; -interface ColumnFile { +interface ColumnFile extends EncryptedFile { Types.NestedField LOCATION = Types.NestedField.required( 160, "location", Types.StringType.get(), "Location of the column file"); @@ -52,21 +51,12 @@ static Types.StructType schema() { return Types.StructType.of(LOCATION, FIELD_IDS, FILE_FORMAT, FILE_SIZE_IN_BYTES, KEY_METADATA); } - /** Returns the location of this column file. */ - String location(); - /** Returns the field IDs contained in this column file. */ Set fieldIds(); /** Returns the format of this column file. */ FileFormat fileFormat(); - /** Returns the total size of this column file in bytes. */ - long fileSizeInBytes(); - - /** Returns encryption key metadata, or null if this column file is not encrypted. */ - ByteBuffer keyMetadata(); - /** Copies this column file. */ ColumnFile copy(); } diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index e6055d9885dc..67c4c6eee0fb 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -79,6 +79,8 @@ class TestV4ManifestReader { static { Mockito.when(UNREAD_MANIFEST_FILE.path()) .thenReturn(FileFormat.PARQUET.addExtension("manifest")); + Mockito.when(UNREAD_MANIFEST_FILE.location()) + .thenReturn(FileFormat.PARQUET.addExtension("manifest")); Mockito.when(UNREAD_MANIFEST_FILE.formatVersion()).thenReturn(4); Mockito.when(UNREAD_MANIFEST_FILE.content()).thenReturn(ManifestContent.DATA); } @@ -1732,6 +1734,7 @@ public void partitionFilterKeepsUnpartitionedFile(FileFormat format) throws IOEx public void validationUnknownManifestFormatFailure() { ManifestFile fileWithoutFormat = Mockito.mock(ManifestFile.class); Mockito.when(fileWithoutFormat.path()).thenReturn("manifest"); + Mockito.when(fileWithoutFormat.location()).thenReturn("manifest"); Mockito.when(fileWithoutFormat.formatVersion()).thenReturn(4); Mockito.when(fileWithoutFormat.content()).thenReturn(ManifestContent.DATA); @@ -1749,6 +1752,7 @@ public void validationUnknownManifestFormatFailure() { public void validationUnknownFormatVersionFailure() { ManifestFile v3File = Mockito.mock(ManifestFile.class); Mockito.when(v3File.path()).thenReturn(FileFormat.PARQUET.addExtension("manifest")); + Mockito.when(v3File.location()).thenReturn(FileFormat.PARQUET.addExtension("manifest")); Mockito.when(v3File.formatVersion()).thenReturn(3); Mockito.when(v3File.content()).thenReturn(ManifestContent.DATA); @@ -1762,6 +1766,7 @@ public void validationUnknownFormatVersionFailure() { public void validationDeleteManifestFailure() { ManifestFile v3File = Mockito.mock(ManifestFile.class); Mockito.when(v3File.path()).thenReturn(FileFormat.PARQUET.addExtension("manifest")); + Mockito.when(v3File.location()).thenReturn(FileFormat.PARQUET.addExtension("manifest")); Mockito.when(v3File.formatVersion()).thenReturn(4); Mockito.when(v3File.content()).thenReturn(ManifestContent.DELETES); @@ -2118,6 +2123,7 @@ private ManifestFile writeManifest( private static ManifestFile v4Manifest(String path) { ManifestFile manifest = mock(ManifestFile.class); when(manifest.path()).thenReturn(path); + when(manifest.location()).thenReturn(path); when(manifest.formatVersion()).thenReturn(FORMAT_VERSION_V4); when(manifest.content()).thenReturn(ManifestContent.DATA); when(manifest.snapshotId()).thenReturn(SNAPSHOT_ID); diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java index 0333f1e45dd4..03fb683b9f3a 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java @@ -30,6 +30,7 @@ import org.apache.iceberg.ContentFile; import org.apache.iceberg.ContentScanTask; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.EncryptedFile; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.Partitioning; import org.apache.iceberg.ScanTask; @@ -185,7 +186,10 @@ private Map inputFiles() { if (lazyInputFiles == null) { this.lazyInputFiles = fileIO.bulkDecrypt( - () -> taskGroup.tasks().stream().flatMap(this::referencedFiles).iterator()); + () -> + taskGroup.tasks().stream() + .flatMap(this::referencedFiles) + .iterator()); } return lazyInputFiles; diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java index 0333f1e45dd4..03fb683b9f3a 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java @@ -30,6 +30,7 @@ import org.apache.iceberg.ContentFile; import org.apache.iceberg.ContentScanTask; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.EncryptedFile; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.Partitioning; import org.apache.iceberg.ScanTask; @@ -185,7 +186,10 @@ private Map inputFiles() { if (lazyInputFiles == null) { this.lazyInputFiles = fileIO.bulkDecrypt( - () -> taskGroup.tasks().stream().flatMap(this::referencedFiles).iterator()); + () -> + taskGroup.tasks().stream() + .flatMap(this::referencedFiles) + .iterator()); } return lazyInputFiles; diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java index 50dc69dd9a55..0784f5a50d06 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java @@ -31,6 +31,7 @@ import org.apache.iceberg.ContentFile; import org.apache.iceberg.ContentScanTask; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.EncryptedFile; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.Partitioning; import org.apache.iceberg.ScanTask; @@ -194,7 +195,10 @@ private Map inputFiles() { if (lazyInputFiles == null) { this.lazyInputFiles = fileIO.bulkDecrypt( - () -> taskGroup.tasks().stream().flatMap(this::referencedFiles).iterator()); + () -> + taskGroup.tasks().stream() + .flatMap(this::referencedFiles) + .iterator()); } return lazyInputFiles; diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java index 50dc69dd9a55..0784f5a50d06 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/BaseReader.java @@ -31,6 +31,7 @@ import org.apache.iceberg.ContentFile; import org.apache.iceberg.ContentScanTask; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.EncryptedFile; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.Partitioning; import org.apache.iceberg.ScanTask; @@ -194,7 +195,10 @@ private Map inputFiles() { if (lazyInputFiles == null) { this.lazyInputFiles = fileIO.bulkDecrypt( - () -> taskGroup.tasks().stream().flatMap(this::referencedFiles).iterator()); + () -> + taskGroup.tasks().stream() + .flatMap(this::referencedFiles) + .iterator()); } return lazyInputFiles;