From f50cb8645408648d23d9f8a4d68206e81fc29534 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Thu, 19 Nov 2026 12:26:00 -0800 Subject: [PATCH 1/3] Add integration tests for reading Spark-encrypted tables Spark writes encrypted v3 tables through its hive catalog with Java's UnitestKMS, and PyIceberg reads them with a matching in-memory KMS. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- dev/spark/Dockerfile | 3 +- dev/spark/spark-defaults.conf | 2 + tests/integration/test_encryption.py | 168 +++++++++++++++++++++++++++ 3 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 tests/integration/test_encryption.py diff --git a/dev/spark/Dockerfile b/dev/spark/Dockerfile index eb94c2938d..4a099d243d 100644 --- a/dev/spark/Dockerfile +++ b/dev/spark/Dockerfile @@ -52,7 +52,8 @@ RUN set -e && \ "org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar" \ "org/apache/hive/hive-standalone-metastore-common/${HIVE_METASTORE_VERSION}/hive-standalone-metastore-common-${HIVE_METASTORE_VERSION}.jar" \ "org/apache/hive/hive-standalone-metastore-client/${HIVE_METASTORE_VERSION}/hive-standalone-metastore-client-${HIVE_METASTORE_VERSION}.jar" \ - "software/amazon/awssdk/bundle/${AWS_SDK_VERSION}/bundle-${AWS_SDK_VERSION}.jar"; \ + "software/amazon/awssdk/bundle/${AWS_SDK_VERSION}/bundle-${AWS_SDK_VERSION}.jar" \ + "org/apache/iceberg/iceberg-core/${ICEBERG_VERSION}/iceberg-core-${ICEBERG_VERSION}-tests.jar"; \ do \ jar_name=$(basename "${jar_path}") && \ curl -fsSL --retry 3 --retry-delay 5 \ diff --git a/dev/spark/spark-defaults.conf b/dev/spark/spark-defaults.conf index 774155d209..11d60f4576 100644 --- a/dev/spark/spark-defaults.conf +++ b/dev/spark/spark-defaults.conf @@ -33,6 +33,8 @@ spark.sql.catalog.hive.uri thrift://hive:9083 spark.sql.catalog.hive.io-impl org.apache.iceberg.aws.s3.S3FileIO spark.sql.catalog.hive.warehouse s3://warehouse/hive/ spark.sql.catalog.hive.s3.endpoint http://object-store:9000 +# Test table encryption with UnitestKMS from iceberg-core's tests jar, an in-memory KMS with static master keys +spark.sql.catalog.hive.encryption.kms-impl org.apache.iceberg.encryption.UnitestKMS # Configure Spark's default session catalog (spark_catalog) to use Iceberg backed by the Hive Metastore spark.sql.catalog.spark_catalog org.apache.iceberg.spark.SparkSessionCatalog diff --git a/tests/integration/test_encryption.py b/tests/integration/test_encryption.py new file mode 100644 index 0000000000..dada822ba3 --- /dev/null +++ b/tests/integration/test_encryption.py @@ -0,0 +1,168 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint:disable=redefined-outer-name + +import pytest +from pyspark.sql import SparkSession + +from pyiceberg.catalog import Catalog +from pyiceberg.encryption.ciphers import SecureKey +from pyiceberg.encryption.kms import KeyManagementClient +from pyiceberg.table import Table +from tests.encryption.memory_kms import MemoryKeyManagementClient + +# Master key keyA of UnitestKMS, the in-memory KMS that Spark uses to write encrypted tables: +# https://github.com/apache/iceberg/blob/212aa894e919ff9fdf89160da4aadcbe6858ac01/core/src/test/java/org/apache/iceberg/encryption/UnitestKMS.java +MASTER_KEY_ID = "keyA" +MASTER_KEY = b"0123456789012345" + +# AES-128 and AES-256 data keys +DATA_KEY_LENGTHS = [16, 32] + +ROWS = [ + {"id": 1, "category": "a", "payload": "alpha"}, + {"id": 2, "category": "b", "payload": None}, + {"id": 3, "category": "a", "payload": "gamma"}, + {"id": 4, "category": "b", "payload": "delta"}, +] +EXTRA_ROWS = [ + {"id": 5, "category": "a", "payload": "epsilon"}, + {"id": 6, "category": "b", "payload": "zeta"}, +] + + +def _create_encrypted_table(spark: SparkSession, identifier: str, data_key_length: int) -> None: + # As of now, only the hive catalog supports KMS encryption in the integration setup. + # Spark's hive catalog is configured with UnitestKMS in dev/spark/spark-defaults.conf. + spark.sql(f"DROP TABLE IF EXISTS hive.{identifier}") + spark.sql( + f""" + CREATE TABLE hive.{identifier} (id bigint, category string, payload string) + USING iceberg + PARTITIONED BY (category) + TBLPROPERTIES ( + 'format-version'='3', + 'encryption.key-id'='{MASTER_KEY_ID}', + 'encryption.data-key-length'='{data_key_length}', + 'write.delete.mode'='merge-on-read' + ) + """ + ) + + +def _load_table(catalog: Catalog, identifier: str, kms: KeyManagementClient) -> Table: + table = catalog.load_table(identifier) + return Table( + identifier=table.name(), + metadata=table.metadata, + metadata_location=table.metadata_location, + io=table.io, + catalog=catalog, + kms_client=kms, + ) + + +@pytest.fixture +def kms() -> KeyManagementClient: + kms = MemoryKeyManagementClient() + kms.add_master_key(MASTER_KEY_ID, SecureKey(MASTER_KEY)) + return kms + + +@pytest.mark.integration +@pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS) +def test_scan_encrypted_table( + spark: SparkSession, session_catalog_hive: Catalog, kms: KeyManagementClient, data_key_length: int +) -> None: + identifier = "default.test_scan_encrypted_table" + _create_encrypted_table(spark, identifier, data_key_length) + spark.createDataFrame(ROWS).writeTo(f"hive.{identifier}").append() + # Delete a row from each partition, so Spark writes a deletion vector per partition + spark.sql(f"DELETE FROM hive.{identifier} WHERE id IN (1, 2)") + table = _load_table(session_catalog_hive, identifier, kms) + + expected = [{"id": 3, "category": "a", "payload": "gamma"}, {"id": 4, "category": "b", "payload": "delta"}] + scan = table.scan() + assert scan.to_arrow().sort_by("id").to_pylist() == expected + assert scan.to_arrow_batch_reader().read_all().sort_by("id").to_pylist() == expected + assert scan.count() == len(expected) + + filtered = table.scan(row_filter="category == 'a'", selected_fields=("id", "payload")).to_arrow() + assert filtered.to_pylist() == [{"id": 3, "payload": "gamma"}] + + +@pytest.mark.integration +@pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS) +def test_scan_encrypted_table_snapshots( + spark: SparkSession, session_catalog_hive: Catalog, kms: KeyManagementClient, data_key_length: int +) -> None: + identifier = "default.test_scan_encrypted_table_snapshots" + _create_encrypted_table(spark, identifier, data_key_length) + spark.createDataFrame(ROWS).writeTo(f"hive.{identifier}").append() + spark.createDataFrame(EXTRA_ROWS).writeTo(f"hive.{identifier}").append() + table = _load_table(session_catalog_hive, identifier, kms) + + first, second = table.snapshots() + assert table.scan(snapshot_id=first.snapshot_id).to_arrow().sort_by("id").to_pylist() == ROWS + assert table.scan(snapshot_id=second.snapshot_id).to_arrow().sort_by("id").to_pylist() == ROWS + EXTRA_ROWS + + +@pytest.mark.integration +@pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS) +def test_scan_encrypted_table_requires_master_key( + spark: SparkSession, session_catalog_hive: Catalog, data_key_length: int +) -> None: + identifier = "default.test_scan_encrypted_table_requires_master_key" + _create_encrypted_table(spark, identifier, data_key_length) + spark.createDataFrame(ROWS).writeTo(f"hive.{identifier}").append() + + with pytest.raises(ValueError, match="no key management client was provided"): + session_catalog_hive.load_table(identifier).scan().to_arrow() + + wrong_kms = MemoryKeyManagementClient() + wrong_kms.add_master_key(MASTER_KEY_ID, SecureKey(bytes(16))) + with pytest.raises(ValueError, match="GCM tag check failed"): + _load_table(session_catalog_hive, identifier, wrong_kms).scan().to_arrow() + + +@pytest.mark.integration +@pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS) +def test_inspect_encrypted_table( + spark: SparkSession, session_catalog_hive: Catalog, kms: KeyManagementClient, data_key_length: int +) -> None: + identifier = "default.test_inspect_encrypted_table" + _create_encrypted_table(spark, identifier, data_key_length) + spark.createDataFrame(ROWS).writeTo(f"hive.{identifier}").append() + # Delete a row from each partition, so Spark writes a deletion vector per partition + spark.sql(f"DELETE FROM hive.{identifier} WHERE id IN (1, 2)") + inspect = _load_table(session_catalog_hive, identifier, kms).inspect + + files = inspect.files() + spark_file_paths = [row.file_path for row in spark.table(f"hive.{identifier}.files").collect()] + assert sorted(files["file_path"].to_pylist()) == sorted(spark_file_paths) + assert all(key_metadata is not None for key_metadata in files["key_metadata"].to_pylist()) + assert sorted(entry["data_file"]["file_path"] for entry in inspect.entries().to_pylist()) == sorted(spark_file_paths) + + spark_manifest_paths = [row.path for row in spark.table(f"hive.{identifier}.manifests").collect()] + assert sorted(inspect.manifests()["path"].to_pylist()) == sorted(spark_manifest_paths) + + # Each partition has two rows, and a deletion vector for one of them + partitions = [ + (row["partition"]["category"], row["record_count"], row["position_delete_record_count"]) + for row in inspect.partitions().to_pylist() + ] + assert sorted(partitions) == [("a", 2, 1), ("b", 2, 1)] From 111a622eb570445900a533d83ca8fb50897e2ce9 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Thu, 19 Nov 2026 12:29:06 -0800 Subject: [PATCH 2/3] Check that Spark encrypts Parquet data files Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/integration/test_encryption.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_encryption.py b/tests/integration/test_encryption.py index dada822ba3..c97eb607ef 100644 --- a/tests/integration/test_encryption.py +++ b/tests/integration/test_encryption.py @@ -16,6 +16,7 @@ # under the License. # pylint:disable=redefined-outer-name +import pyarrow.parquet as pq import pytest from pyspark.sql import SparkSession @@ -124,7 +125,7 @@ def test_scan_encrypted_table_snapshots( @pytest.mark.integration @pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS) def test_scan_encrypted_table_requires_master_key( - spark: SparkSession, session_catalog_hive: Catalog, data_key_length: int + spark: SparkSession, session_catalog_hive: Catalog, kms: KeyManagementClient, data_key_length: int ) -> None: identifier = "default.test_scan_encrypted_table_requires_master_key" _create_encrypted_table(spark, identifier, data_key_length) @@ -138,6 +139,15 @@ def test_scan_encrypted_table_requires_master_key( with pytest.raises(ValueError, match="GCM tag check failed"): _load_table(session_catalog_hive, identifier, wrong_kms).scan().to_arrow() + # Spark also encrypts the Parquet data files, so they can't be read without their keys + table = _load_table(session_catalog_hive, identifier, kms) + data_files = [task.file.file_path for task in table.scan().plan_files()] + assert data_files + for data_file in data_files: + with table.io.new_input(data_file).open() as stream: + with pytest.raises(OSError, match="no decryption found"): + pq.read_table(stream) + @pytest.mark.integration @pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS) From aab1459fc593631c619cef6324d89d16ec94cdf7 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Thu, 19 Nov 2026 15:18:20 -0800 Subject: [PATCH 3/3] Use AWS KMS backed by moto for encryption integration tests Adds AwsKeyManagementClient and a moto KMS service, replacing UnitestKMS and the iceberg-core tests jar in the Spark image. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- dev/docker-compose-integration.yml | 9 ++ dev/spark/Dockerfile | 3 +- dev/spark/spark-defaults.conf | 5 +- pyiceberg/encryption/aws_kms.py | 56 ++++++++++++ tests/encryption/test_kms.py | 16 ++++ tests/integration/test_encryption.py | 125 +++++++++++++-------------- 6 files changed, 143 insertions(+), 71 deletions(-) create mode 100644 pyiceberg/encryption/aws_kms.py diff --git a/dev/docker-compose-integration.yml b/dev/docker-compose-integration.yml index dd3f416420..e30d963a3a 100644 --- a/dev/docker-compose-integration.yml +++ b/dev/docker-compose-integration.yml @@ -101,6 +101,15 @@ services: command: - -c - "aws s3 rb s3://warehouse --force 2>/dev/null; aws s3 mb s3://warehouse" + kms: + image: motoserver/moto:5.0.3 + container_name: pyiceberg-kms + networks: + iceberg_net: + # Use port 5001 instead of default 5000 to avoid conflict with macOS AirPlay Receiver. + command: ["-p", "5001"] + ports: + - 5001:5001 hive: image: pyiceberg-hive:latest build: diff --git a/dev/spark/Dockerfile b/dev/spark/Dockerfile index 4a099d243d..eb94c2938d 100644 --- a/dev/spark/Dockerfile +++ b/dev/spark/Dockerfile @@ -52,8 +52,7 @@ RUN set -e && \ "org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar" \ "org/apache/hive/hive-standalone-metastore-common/${HIVE_METASTORE_VERSION}/hive-standalone-metastore-common-${HIVE_METASTORE_VERSION}.jar" \ "org/apache/hive/hive-standalone-metastore-client/${HIVE_METASTORE_VERSION}/hive-standalone-metastore-client-${HIVE_METASTORE_VERSION}.jar" \ - "software/amazon/awssdk/bundle/${AWS_SDK_VERSION}/bundle-${AWS_SDK_VERSION}.jar" \ - "org/apache/iceberg/iceberg-core/${ICEBERG_VERSION}/iceberg-core-${ICEBERG_VERSION}-tests.jar"; \ + "software/amazon/awssdk/bundle/${AWS_SDK_VERSION}/bundle-${AWS_SDK_VERSION}.jar"; \ do \ jar_name=$(basename "${jar_path}") && \ curl -fsSL --retry 3 --retry-delay 5 \ diff --git a/dev/spark/spark-defaults.conf b/dev/spark/spark-defaults.conf index 11d60f4576..c78afd3244 100644 --- a/dev/spark/spark-defaults.conf +++ b/dev/spark/spark-defaults.conf @@ -33,8 +33,9 @@ spark.sql.catalog.hive.uri thrift://hive:9083 spark.sql.catalog.hive.io-impl org.apache.iceberg.aws.s3.S3FileIO spark.sql.catalog.hive.warehouse s3://warehouse/hive/ spark.sql.catalog.hive.s3.endpoint http://object-store:9000 -# Test table encryption with UnitestKMS from iceberg-core's tests jar, an in-memory KMS with static master keys -spark.sql.catalog.hive.encryption.kms-impl org.apache.iceberg.encryption.UnitestKMS +# Table encryption uses AWS KMS, mocked by moto in the kms service +spark.sql.catalog.hive.encryption.kms-type aws +spark.sql.catalog.hive.kms.endpoint http://kms:5001 # Configure Spark's default session catalog (spark_catalog) to use Iceberg backed by the Hive Metastore spark.sql.catalog.spark_catalog org.apache.iceberg.spark.SparkSessionCatalog diff --git a/pyiceberg/encryption/aws_kms.py b/pyiceberg/encryption/aws_kms.py new file mode 100644 index 0000000000..8d50938163 --- /dev/null +++ b/pyiceberg/encryption/aws_kms.py @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""AWS KMS key management client for table encryption.""" + +from __future__ import annotations + +import boto3 + +from pyiceberg.encryption.kms import KeyManagementClient +from pyiceberg.io import AWS_ACCESS_KEY_ID, AWS_PROFILE_NAME, AWS_REGION, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN +from pyiceberg.typedef import EMPTY_DICT, Properties +from pyiceberg.utils.transient import _TransientStateMixin + +KMS_ENDPOINT = "kms.endpoint" + + +class AwsKeyManagementClient(_TransientStateMixin, KeyManagementClient): + """Wraps and unwraps keys with master keys held by AWS KMS.""" + + _transient_attrs = ("_client",) + + def __init__(self, properties: Properties = EMPTY_DICT) -> None: + super().__init__(properties) + self._init_transient() + + def _init_transient(self) -> None: + session = boto3.Session( + profile_name=self.properties.get(AWS_PROFILE_NAME), + region_name=self.properties.get(AWS_REGION), + aws_access_key_id=self.properties.get(AWS_ACCESS_KEY_ID), + aws_secret_access_key=self.properties.get(AWS_SECRET_ACCESS_KEY), + aws_session_token=self.properties.get(AWS_SESSION_TOKEN), + ) + self._client = session.client("kms", endpoint_url=self.properties.get(KMS_ENDPOINT)) + + def wrap_key(self, key: bytes, wrapping_key_id: str) -> bytes: + """Encrypt `key` with the KMS key `wrapping_key_id`.""" + return self._client.encrypt(KeyId=wrapping_key_id, Plaintext=key)["CiphertextBlob"] + + def unwrap_key(self, wrapped_key: bytes, wrapping_key_id: str) -> bytes: + """Decrypt `wrapped_key` with the KMS key `wrapping_key_id`.""" + return self._client.decrypt(KeyId=wrapping_key_id, CiphertextBlob=wrapped_key)["Plaintext"] diff --git a/tests/encryption/test_kms.py b/tests/encryption/test_kms.py index b0c80ecf1c..95820d09b5 100644 --- a/tests/encryption/test_kms.py +++ b/tests/encryption/test_kms.py @@ -15,9 +15,14 @@ # specific language governing permissions and limitations # under the License. +import pickle + +import boto3 import pytest from memory_kms import MemoryKeyManagementClient +from moto import mock_aws +from pyiceberg.encryption.aws_kms import AwsKeyManagementClient from pyiceberg.encryption.ciphers import AesGcmCipher, AesKeySize, SecureKey from pyiceberg.encryption.kms import GeneratedKey, KeyManagementClient @@ -145,3 +150,14 @@ def test_repr_redacts_master_keys(kms: MemoryKeyManagementClient) -> None: assert repr(kms) == "MemoryKeyManagementClient(master_key_size=, key_count=2)" assert repr(MASTER_KEY.key) not in repr(kms) + + +@mock_aws +def test_aws_kms_wrap_unwrap_round_trip(_aws_credentials: None) -> None: + master_key_id = boto3.client("kms").create_key()["KeyMetadata"]["KeyId"] + kms = AwsKeyManagementClient() + + wrapped = kms.wrap_key(DEK, master_key_id) + + assert kms.unwrap_key(wrapped, master_key_id) == DEK + assert pickle.loads(pickle.dumps(kms)).unwrap_key(wrapped, master_key_id) == DEK diff --git a/tests/integration/test_encryption.py b/tests/integration/test_encryption.py index c97eb607ef..3786468bae 100644 --- a/tests/integration/test_encryption.py +++ b/tests/integration/test_encryption.py @@ -14,22 +14,15 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# pylint:disable=redefined-outer-name - +import boto3 import pyarrow.parquet as pq import pytest from pyspark.sql import SparkSession from pyiceberg.catalog import Catalog -from pyiceberg.encryption.ciphers import SecureKey -from pyiceberg.encryption.kms import KeyManagementClient +from pyiceberg.encryption.aws_kms import KMS_ENDPOINT, AwsKeyManagementClient +from pyiceberg.io import AWS_ACCESS_KEY_ID, AWS_REGION, AWS_SECRET_ACCESS_KEY from pyiceberg.table import Table -from tests.encryption.memory_kms import MemoryKeyManagementClient - -# Master key keyA of UnitestKMS, the in-memory KMS that Spark uses to write encrypted tables: -# https://github.com/apache/iceberg/blob/212aa894e919ff9fdf89160da4aadcbe6858ac01/core/src/test/java/org/apache/iceberg/encryption/UnitestKMS.java -MASTER_KEY_ID = "keyA" -MASTER_KEY = b"0123456789012345" # AES-128 and AES-256 data keys DATA_KEY_LENGTHS = [16, 32] @@ -46,55 +39,15 @@ ] -def _create_encrypted_table(spark: SparkSession, identifier: str, data_key_length: int) -> None: - # As of now, only the hive catalog supports KMS encryption in the integration setup. - # Spark's hive catalog is configured with UnitestKMS in dev/spark/spark-defaults.conf. - spark.sql(f"DROP TABLE IF EXISTS hive.{identifier}") - spark.sql( - f""" - CREATE TABLE hive.{identifier} (id bigint, category string, payload string) - USING iceberg - PARTITIONED BY (category) - TBLPROPERTIES ( - 'format-version'='3', - 'encryption.key-id'='{MASTER_KEY_ID}', - 'encryption.data-key-length'='{data_key_length}', - 'write.delete.mode'='merge-on-read' - ) - """ - ) - - -def _load_table(catalog: Catalog, identifier: str, kms: KeyManagementClient) -> Table: - table = catalog.load_table(identifier) - return Table( - identifier=table.name(), - metadata=table.metadata, - metadata_location=table.metadata_location, - io=table.io, - catalog=catalog, - kms_client=kms, - ) - - -@pytest.fixture -def kms() -> KeyManagementClient: - kms = MemoryKeyManagementClient() - kms.add_master_key(MASTER_KEY_ID, SecureKey(MASTER_KEY)) - return kms - - @pytest.mark.integration @pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS) -def test_scan_encrypted_table( - spark: SparkSession, session_catalog_hive: Catalog, kms: KeyManagementClient, data_key_length: int -) -> None: +def test_scan_encrypted_table(spark: SparkSession, session_catalog_hive: Catalog, data_key_length: int) -> None: identifier = "default.test_scan_encrypted_table" _create_encrypted_table(spark, identifier, data_key_length) spark.createDataFrame(ROWS).writeTo(f"hive.{identifier}").append() # Delete a row from each partition, so Spark writes a deletion vector per partition spark.sql(f"DELETE FROM hive.{identifier} WHERE id IN (1, 2)") - table = _load_table(session_catalog_hive, identifier, kms) + table = _load_table(session_catalog_hive, identifier) expected = [{"id": 3, "category": "a", "payload": "gamma"}, {"id": 4, "category": "b", "payload": "delta"}] scan = table.scan() @@ -108,14 +61,12 @@ def test_scan_encrypted_table( @pytest.mark.integration @pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS) -def test_scan_encrypted_table_snapshots( - spark: SparkSession, session_catalog_hive: Catalog, kms: KeyManagementClient, data_key_length: int -) -> None: +def test_scan_encrypted_table_snapshots(spark: SparkSession, session_catalog_hive: Catalog, data_key_length: int) -> None: identifier = "default.test_scan_encrypted_table_snapshots" _create_encrypted_table(spark, identifier, data_key_length) spark.createDataFrame(ROWS).writeTo(f"hive.{identifier}").append() spark.createDataFrame(EXTRA_ROWS).writeTo(f"hive.{identifier}").append() - table = _load_table(session_catalog_hive, identifier, kms) + table = _load_table(session_catalog_hive, identifier) first, second = table.snapshots() assert table.scan(snapshot_id=first.snapshot_id).to_arrow().sort_by("id").to_pylist() == ROWS @@ -125,7 +76,7 @@ def test_scan_encrypted_table_snapshots( @pytest.mark.integration @pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS) def test_scan_encrypted_table_requires_master_key( - spark: SparkSession, session_catalog_hive: Catalog, kms: KeyManagementClient, data_key_length: int + spark: SparkSession, session_catalog_hive: Catalog, data_key_length: int ) -> None: identifier = "default.test_scan_encrypted_table_requires_master_key" _create_encrypted_table(spark, identifier, data_key_length) @@ -134,13 +85,8 @@ def test_scan_encrypted_table_requires_master_key( with pytest.raises(ValueError, match="no key management client was provided"): session_catalog_hive.load_table(identifier).scan().to_arrow() - wrong_kms = MemoryKeyManagementClient() - wrong_kms.add_master_key(MASTER_KEY_ID, SecureKey(bytes(16))) - with pytest.raises(ValueError, match="GCM tag check failed"): - _load_table(session_catalog_hive, identifier, wrong_kms).scan().to_arrow() - # Spark also encrypts the Parquet data files, so they can't be read without their keys - table = _load_table(session_catalog_hive, identifier, kms) + table = _load_table(session_catalog_hive, identifier) data_files = [task.file.file_path for task in table.scan().plan_files()] assert data_files for data_file in data_files: @@ -151,15 +97,13 @@ def test_scan_encrypted_table_requires_master_key( @pytest.mark.integration @pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS) -def test_inspect_encrypted_table( - spark: SparkSession, session_catalog_hive: Catalog, kms: KeyManagementClient, data_key_length: int -) -> None: +def test_inspect_encrypted_table(spark: SparkSession, session_catalog_hive: Catalog, data_key_length: int) -> None: identifier = "default.test_inspect_encrypted_table" _create_encrypted_table(spark, identifier, data_key_length) spark.createDataFrame(ROWS).writeTo(f"hive.{identifier}").append() # Delete a row from each partition, so Spark writes a deletion vector per partition spark.sql(f"DELETE FROM hive.{identifier} WHERE id IN (1, 2)") - inspect = _load_table(session_catalog_hive, identifier, kms).inspect + inspect = _load_table(session_catalog_hive, identifier).inspect files = inspect.files() spark_file_paths = [row.file_path for row in spark.table(f"hive.{identifier}.files").collect()] @@ -176,3 +120,50 @@ def test_inspect_encrypted_table( for row in inspect.partitions().to_pylist() ] assert sorted(partitions) == [("a", 2, 1), ("b", 2, 1)] + + +def _create_encrypted_table(spark: SparkSession, identifier: str, data_key_length: int) -> None: + # As of Iceberg Java 1.12, only the hive catalog supports table encryption. + # Spark's hive catalog uses the mock KMS configured in dev/spark/spark-defaults.conf. + kms = boto3.client( + "kms", + endpoint_url="http://localhost:5001", + region_name="us-east-1", + aws_access_key_id="admin", + aws_secret_access_key="password", + ) + master_key_id = kms.create_key()["KeyMetadata"]["KeyId"] + + spark.sql(f"DROP TABLE IF EXISTS hive.{identifier}") + spark.sql( + f""" + CREATE TABLE hive.{identifier} (id bigint, category string, payload string) + USING iceberg + PARTITIONED BY (category) + TBLPROPERTIES ( + 'format-version'='3', + 'encryption.key-id'='{master_key_id}', + 'encryption.data-key-length'='{data_key_length}', + 'write.delete.mode'='merge-on-read' + ) + """ + ) + + +def _load_table(catalog: Catalog, identifier: str) -> Table: + table = catalog.load_table(identifier) + return Table( + identifier=table.name(), + metadata=table.metadata, + metadata_location=table.metadata_location, + io=table.io, + catalog=catalog, + kms_client=AwsKeyManagementClient( + { + KMS_ENDPOINT: "http://localhost:5001", + AWS_REGION: "us-east-1", + AWS_ACCESS_KEY_ID: "admin", + AWS_SECRET_ACCESS_KEY: "password", + } + ), + )