diff --git a/dev/docker-compose-integration.yml b/dev/docker-compose-integration.yml index dd3f416420..e30d963a3a 100644 --- a/dev/docker-compose-integration.yml +++ b/dev/docker-compose-integration.yml @@ -101,6 +101,15 @@ services: command: - -c - "aws s3 rb s3://warehouse --force 2>/dev/null; aws s3 mb s3://warehouse" + kms: + image: motoserver/moto:5.0.3 + container_name: pyiceberg-kms + networks: + iceberg_net: + # Use port 5001 instead of default 5000 to avoid conflict with macOS AirPlay Receiver. + command: ["-p", "5001"] + ports: + - 5001:5001 hive: image: pyiceberg-hive:latest build: diff --git a/dev/spark/spark-defaults.conf b/dev/spark/spark-defaults.conf index 774155d209..c78afd3244 100644 --- a/dev/spark/spark-defaults.conf +++ b/dev/spark/spark-defaults.conf @@ -33,6 +33,9 @@ spark.sql.catalog.hive.uri thrift://hive:9083 spark.sql.catalog.hive.io-impl org.apache.iceberg.aws.s3.S3FileIO spark.sql.catalog.hive.warehouse s3://warehouse/hive/ spark.sql.catalog.hive.s3.endpoint http://object-store:9000 +# Table encryption uses AWS KMS, mocked by moto in the kms service +spark.sql.catalog.hive.encryption.kms-type aws +spark.sql.catalog.hive.kms.endpoint http://kms:5001 # Configure Spark's default session catalog (spark_catalog) to use Iceberg backed by the Hive Metastore spark.sql.catalog.spark_catalog org.apache.iceberg.spark.SparkSessionCatalog diff --git a/pyiceberg/encryption/aws_kms.py b/pyiceberg/encryption/aws_kms.py new file mode 100644 index 0000000000..8d50938163 --- /dev/null +++ b/pyiceberg/encryption/aws_kms.py @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""AWS KMS key management client for table encryption.""" + +from __future__ import annotations + +import boto3 + +from pyiceberg.encryption.kms import KeyManagementClient +from pyiceberg.io import AWS_ACCESS_KEY_ID, AWS_PROFILE_NAME, AWS_REGION, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN +from pyiceberg.typedef import EMPTY_DICT, Properties +from pyiceberg.utils.transient import _TransientStateMixin + +KMS_ENDPOINT = "kms.endpoint" + + +class AwsKeyManagementClient(_TransientStateMixin, KeyManagementClient): + """Wraps and unwraps keys with master keys held by AWS KMS.""" + + _transient_attrs = ("_client",) + + def __init__(self, properties: Properties = EMPTY_DICT) -> None: + super().__init__(properties) + self._init_transient() + + def _init_transient(self) -> None: + session = boto3.Session( + profile_name=self.properties.get(AWS_PROFILE_NAME), + region_name=self.properties.get(AWS_REGION), + aws_access_key_id=self.properties.get(AWS_ACCESS_KEY_ID), + aws_secret_access_key=self.properties.get(AWS_SECRET_ACCESS_KEY), + aws_session_token=self.properties.get(AWS_SESSION_TOKEN), + ) + self._client = session.client("kms", endpoint_url=self.properties.get(KMS_ENDPOINT)) + + def wrap_key(self, key: bytes, wrapping_key_id: str) -> bytes: + """Encrypt `key` with the KMS key `wrapping_key_id`.""" + return self._client.encrypt(KeyId=wrapping_key_id, Plaintext=key)["CiphertextBlob"] + + def unwrap_key(self, wrapped_key: bytes, wrapping_key_id: str) -> bytes: + """Decrypt `wrapped_key` with the KMS key `wrapping_key_id`.""" + return self._client.decrypt(KeyId=wrapping_key_id, CiphertextBlob=wrapped_key)["Plaintext"] diff --git a/tests/encryption/test_kms.py b/tests/encryption/test_kms.py index b0c80ecf1c..95820d09b5 100644 --- a/tests/encryption/test_kms.py +++ b/tests/encryption/test_kms.py @@ -15,9 +15,14 @@ # specific language governing permissions and limitations # under the License. +import pickle + +import boto3 import pytest from memory_kms import MemoryKeyManagementClient +from moto import mock_aws +from pyiceberg.encryption.aws_kms import AwsKeyManagementClient from pyiceberg.encryption.ciphers import AesGcmCipher, AesKeySize, SecureKey from pyiceberg.encryption.kms import GeneratedKey, KeyManagementClient @@ -145,3 +150,14 @@ def test_repr_redacts_master_keys(kms: MemoryKeyManagementClient) -> None: assert repr(kms) == "MemoryKeyManagementClient(master_key_size=, key_count=2)" assert repr(MASTER_KEY.key) not in repr(kms) + + +@mock_aws +def test_aws_kms_wrap_unwrap_round_trip(_aws_credentials: None) -> None: + master_key_id = boto3.client("kms").create_key()["KeyMetadata"]["KeyId"] + kms = AwsKeyManagementClient() + + wrapped = kms.wrap_key(DEK, master_key_id) + + assert kms.unwrap_key(wrapped, master_key_id) == DEK + assert pickle.loads(pickle.dumps(kms)).unwrap_key(wrapped, master_key_id) == DEK diff --git a/tests/integration/test_encryption.py b/tests/integration/test_encryption.py new file mode 100644 index 0000000000..3786468bae --- /dev/null +++ b/tests/integration/test_encryption.py @@ -0,0 +1,169 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import boto3 +import pyarrow.parquet as pq +import pytest +from pyspark.sql import SparkSession + +from pyiceberg.catalog import Catalog +from pyiceberg.encryption.aws_kms import KMS_ENDPOINT, AwsKeyManagementClient +from pyiceberg.io import AWS_ACCESS_KEY_ID, AWS_REGION, AWS_SECRET_ACCESS_KEY +from pyiceberg.table import Table + +# AES-128 and AES-256 data keys +DATA_KEY_LENGTHS = [16, 32] + +ROWS = [ + {"id": 1, "category": "a", "payload": "alpha"}, + {"id": 2, "category": "b", "payload": None}, + {"id": 3, "category": "a", "payload": "gamma"}, + {"id": 4, "category": "b", "payload": "delta"}, +] +EXTRA_ROWS = [ + {"id": 5, "category": "a", "payload": "epsilon"}, + {"id": 6, "category": "b", "payload": "zeta"}, +] + + +@pytest.mark.integration +@pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS) +def test_scan_encrypted_table(spark: SparkSession, session_catalog_hive: Catalog, data_key_length: int) -> None: + identifier = "default.test_scan_encrypted_table" + _create_encrypted_table(spark, identifier, data_key_length) + spark.createDataFrame(ROWS).writeTo(f"hive.{identifier}").append() + # Delete a row from each partition, so Spark writes a deletion vector per partition + spark.sql(f"DELETE FROM hive.{identifier} WHERE id IN (1, 2)") + table = _load_table(session_catalog_hive, identifier) + + expected = [{"id": 3, "category": "a", "payload": "gamma"}, {"id": 4, "category": "b", "payload": "delta"}] + scan = table.scan() + assert scan.to_arrow().sort_by("id").to_pylist() == expected + assert scan.to_arrow_batch_reader().read_all().sort_by("id").to_pylist() == expected + assert scan.count() == len(expected) + + filtered = table.scan(row_filter="category == 'a'", selected_fields=("id", "payload")).to_arrow() + assert filtered.to_pylist() == [{"id": 3, "payload": "gamma"}] + + +@pytest.mark.integration +@pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS) +def test_scan_encrypted_table_snapshots(spark: SparkSession, session_catalog_hive: Catalog, data_key_length: int) -> None: + identifier = "default.test_scan_encrypted_table_snapshots" + _create_encrypted_table(spark, identifier, data_key_length) + spark.createDataFrame(ROWS).writeTo(f"hive.{identifier}").append() + spark.createDataFrame(EXTRA_ROWS).writeTo(f"hive.{identifier}").append() + table = _load_table(session_catalog_hive, identifier) + + first, second = table.snapshots() + assert table.scan(snapshot_id=first.snapshot_id).to_arrow().sort_by("id").to_pylist() == ROWS + assert table.scan(snapshot_id=second.snapshot_id).to_arrow().sort_by("id").to_pylist() == ROWS + EXTRA_ROWS + + +@pytest.mark.integration +@pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS) +def test_scan_encrypted_table_requires_master_key( + spark: SparkSession, session_catalog_hive: Catalog, data_key_length: int +) -> None: + identifier = "default.test_scan_encrypted_table_requires_master_key" + _create_encrypted_table(spark, identifier, data_key_length) + spark.createDataFrame(ROWS).writeTo(f"hive.{identifier}").append() + + with pytest.raises(ValueError, match="no key management client was provided"): + session_catalog_hive.load_table(identifier).scan().to_arrow() + + # Spark also encrypts the Parquet data files, so they can't be read without their keys + table = _load_table(session_catalog_hive, identifier) + data_files = [task.file.file_path for task in table.scan().plan_files()] + assert data_files + for data_file in data_files: + with table.io.new_input(data_file).open() as stream: + with pytest.raises(OSError, match="no decryption found"): + pq.read_table(stream) + + +@pytest.mark.integration +@pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS) +def test_inspect_encrypted_table(spark: SparkSession, session_catalog_hive: Catalog, data_key_length: int) -> None: + identifier = "default.test_inspect_encrypted_table" + _create_encrypted_table(spark, identifier, data_key_length) + spark.createDataFrame(ROWS).writeTo(f"hive.{identifier}").append() + # Delete a row from each partition, so Spark writes a deletion vector per partition + spark.sql(f"DELETE FROM hive.{identifier} WHERE id IN (1, 2)") + inspect = _load_table(session_catalog_hive, identifier).inspect + + files = inspect.files() + spark_file_paths = [row.file_path for row in spark.table(f"hive.{identifier}.files").collect()] + assert sorted(files["file_path"].to_pylist()) == sorted(spark_file_paths) + assert all(key_metadata is not None for key_metadata in files["key_metadata"].to_pylist()) + assert sorted(entry["data_file"]["file_path"] for entry in inspect.entries().to_pylist()) == sorted(spark_file_paths) + + spark_manifest_paths = [row.path for row in spark.table(f"hive.{identifier}.manifests").collect()] + assert sorted(inspect.manifests()["path"].to_pylist()) == sorted(spark_manifest_paths) + + # Each partition has two rows, and a deletion vector for one of them + partitions = [ + (row["partition"]["category"], row["record_count"], row["position_delete_record_count"]) + for row in inspect.partitions().to_pylist() + ] + assert sorted(partitions) == [("a", 2, 1), ("b", 2, 1)] + + +def _create_encrypted_table(spark: SparkSession, identifier: str, data_key_length: int) -> None: + # As of Iceberg Java 1.12, only the hive catalog supports table encryption. + # Spark's hive catalog uses the mock KMS configured in dev/spark/spark-defaults.conf. + kms = boto3.client( + "kms", + endpoint_url="http://localhost:5001", + region_name="us-east-1", + aws_access_key_id="admin", + aws_secret_access_key="password", + ) + master_key_id = kms.create_key()["KeyMetadata"]["KeyId"] + + spark.sql(f"DROP TABLE IF EXISTS hive.{identifier}") + spark.sql( + f""" + CREATE TABLE hive.{identifier} (id bigint, category string, payload string) + USING iceberg + PARTITIONED BY (category) + TBLPROPERTIES ( + 'format-version'='3', + 'encryption.key-id'='{master_key_id}', + 'encryption.data-key-length'='{data_key_length}', + 'write.delete.mode'='merge-on-read' + ) + """ + ) + + +def _load_table(catalog: Catalog, identifier: str) -> Table: + table = catalog.load_table(identifier) + return Table( + identifier=table.name(), + metadata=table.metadata, + metadata_location=table.metadata_location, + io=table.io, + catalog=catalog, + kms_client=AwsKeyManagementClient( + { + KMS_ENDPOINT: "http://localhost:5001", + AWS_REGION: "us-east-1", + AWS_ACCESS_KEY_ID: "admin", + AWS_SECRET_ACCESS_KEY: "password", + } + ), + )