Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions dev/docker-compose-integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,15 @@ services:
command:
- -c
- "aws s3 rb s3://warehouse --force 2>/dev/null; aws s3 mb s3://warehouse"
kms:
image: motoserver/moto:5.0.3
container_name: pyiceberg-kms
networks:
iceberg_net:
# Use port 5001 instead of default 5000 to avoid conflict with macOS AirPlay Receiver.
command: ["-p", "5001"]
ports:
- 5001:5001
hive:
image: pyiceberg-hive:latest
build:
Expand Down
3 changes: 3 additions & 0 deletions dev/spark/spark-defaults.conf
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@ spark.sql.catalog.hive.uri thrift://hive:9083
spark.sql.catalog.hive.io-impl org.apache.iceberg.aws.s3.S3FileIO
spark.sql.catalog.hive.warehouse s3://warehouse/hive/
spark.sql.catalog.hive.s3.endpoint http://object-store:9000
# Table encryption uses AWS KMS, mocked by moto in the kms service
spark.sql.catalog.hive.encryption.kms-type aws
spark.sql.catalog.hive.kms.endpoint http://kms:5001
Comment on lines +36 to +38

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

using moto to mock kms service.

only using hive catalog because its the only one supporting table encryption as of Iceberg 1.12


# Configure Spark's default session catalog (spark_catalog) to use Iceberg backed by the Hive Metastore
spark.sql.catalog.spark_catalog org.apache.iceberg.spark.SparkSessionCatalog
Expand Down
56 changes: 56 additions & 0 deletions pyiceberg/encryption/aws_kms.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""AWS KMS key management client for table encryption."""

from __future__ import annotations

import boto3

from pyiceberg.encryption.kms import KeyManagementClient
from pyiceberg.io import AWS_ACCESS_KEY_ID, AWS_PROFILE_NAME, AWS_REGION, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN
from pyiceberg.typedef import EMPTY_DICT, Properties
from pyiceberg.utils.transient import _TransientStateMixin

KMS_ENDPOINT = "kms.endpoint"


class AwsKeyManagementClient(_TransientStateMixin, KeyManagementClient):

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this add an aws kms client.

was using this for testing, but then thought we can just reuse it

"""Wraps and unwraps keys with master keys held by AWS KMS."""

_transient_attrs = ("_client",)

def __init__(self, properties: Properties = EMPTY_DICT) -> None:
super().__init__(properties)
self._init_transient()

def _init_transient(self) -> None:
session = boto3.Session(
profile_name=self.properties.get(AWS_PROFILE_NAME),
region_name=self.properties.get(AWS_REGION),
aws_access_key_id=self.properties.get(AWS_ACCESS_KEY_ID),
aws_secret_access_key=self.properties.get(AWS_SECRET_ACCESS_KEY),
aws_session_token=self.properties.get(AWS_SESSION_TOKEN),
)
self._client = session.client("kms", endpoint_url=self.properties.get(KMS_ENDPOINT))

def wrap_key(self, key: bytes, wrapping_key_id: str) -> bytes:
"""Encrypt `key` with the KMS key `wrapping_key_id`."""
return self._client.encrypt(KeyId=wrapping_key_id, Plaintext=key)["CiphertextBlob"]

def unwrap_key(self, wrapped_key: bytes, wrapping_key_id: str) -> bytes:
"""Decrypt `wrapped_key` with the KMS key `wrapping_key_id`."""
return self._client.decrypt(KeyId=wrapping_key_id, CiphertextBlob=wrapped_key)["Plaintext"]
16 changes: 16 additions & 0 deletions tests/encryption/test_kms.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,14 @@
# specific language governing permissions and limitations
# under the License.

import pickle

import boto3
import pytest
from memory_kms import MemoryKeyManagementClient
from moto import mock_aws

from pyiceberg.encryption.aws_kms import AwsKeyManagementClient
from pyiceberg.encryption.ciphers import AesGcmCipher, AesKeySize, SecureKey
from pyiceberg.encryption.kms import GeneratedKey, KeyManagementClient

Expand Down Expand Up @@ -145,3 +150,14 @@ def test_repr_redacts_master_keys(kms: MemoryKeyManagementClient) -> None:

assert repr(kms) == "MemoryKeyManagementClient(master_key_size=<AesKeySize.BITS_128: 128>, key_count=2)"
assert repr(MASTER_KEY.key) not in repr(kms)


@mock_aws
def test_aws_kms_wrap_unwrap_round_trip(_aws_credentials: None) -> None:
master_key_id = boto3.client("kms").create_key()["KeyMetadata"]["KeyId"]
kms = AwsKeyManagementClient()

wrapped = kms.wrap_key(DEK, master_key_id)

assert kms.unwrap_key(wrapped, master_key_id) == DEK
assert pickle.loads(pickle.dumps(kms)).unwrap_key(wrapped, master_key_id) == DEK
169 changes: 169 additions & 0 deletions tests/integration/test_encryption.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import boto3
import pyarrow.parquet as pq
import pytest
from pyspark.sql import SparkSession

from pyiceberg.catalog import Catalog
from pyiceberg.encryption.aws_kms import KMS_ENDPOINT, AwsKeyManagementClient
from pyiceberg.io import AWS_ACCESS_KEY_ID, AWS_REGION, AWS_SECRET_ACCESS_KEY
from pyiceberg.table import Table

# AES-128 and AES-256 data keys
DATA_KEY_LENGTHS = [16, 32]
Comment on lines +27 to +28

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

parameterize tests by the key lengths


ROWS = [
{"id": 1, "category": "a", "payload": "alpha"},
{"id": 2, "category": "b", "payload": None},
{"id": 3, "category": "a", "payload": "gamma"},
{"id": 4, "category": "b", "payload": "delta"},
]
EXTRA_ROWS = [
{"id": 5, "category": "a", "payload": "epsilon"},
{"id": 6, "category": "b", "payload": "zeta"},
]


@pytest.mark.integration
@pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS)
def test_scan_encrypted_table(spark: SparkSession, session_catalog_hive: Catalog, data_key_length: int) -> None:
identifier = "default.test_scan_encrypted_table"
_create_encrypted_table(spark, identifier, data_key_length)
spark.createDataFrame(ROWS).writeTo(f"hive.{identifier}").append()
# Delete a row from each partition, so Spark writes a deletion vector per partition
spark.sql(f"DELETE FROM hive.{identifier} WHERE id IN (1, 2)")
table = _load_table(session_catalog_hive, identifier)

expected = [{"id": 3, "category": "a", "payload": "gamma"}, {"id": 4, "category": "b", "payload": "delta"}]
scan = table.scan()
assert scan.to_arrow().sort_by("id").to_pylist() == expected
assert scan.to_arrow_batch_reader().read_all().sort_by("id").to_pylist() == expected
assert scan.count() == len(expected)

filtered = table.scan(row_filter="category == 'a'", selected_fields=("id", "payload")).to_arrow()
assert filtered.to_pylist() == [{"id": 3, "payload": "gamma"}]


@pytest.mark.integration
@pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS)
def test_scan_encrypted_table_snapshots(spark: SparkSession, session_catalog_hive: Catalog, data_key_length: int) -> None:
identifier = "default.test_scan_encrypted_table_snapshots"
_create_encrypted_table(spark, identifier, data_key_length)
spark.createDataFrame(ROWS).writeTo(f"hive.{identifier}").append()
spark.createDataFrame(EXTRA_ROWS).writeTo(f"hive.{identifier}").append()
table = _load_table(session_catalog_hive, identifier)

first, second = table.snapshots()
assert table.scan(snapshot_id=first.snapshot_id).to_arrow().sort_by("id").to_pylist() == ROWS
assert table.scan(snapshot_id=second.snapshot_id).to_arrow().sort_by("id").to_pylist() == ROWS + EXTRA_ROWS


@pytest.mark.integration
@pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS)
def test_scan_encrypted_table_requires_master_key(
spark: SparkSession, session_catalog_hive: Catalog, data_key_length: int
) -> None:
identifier = "default.test_scan_encrypted_table_requires_master_key"
_create_encrypted_table(spark, identifier, data_key_length)
spark.createDataFrame(ROWS).writeTo(f"hive.{identifier}").append()

with pytest.raises(ValueError, match="no key management client was provided"):
session_catalog_hive.load_table(identifier).scan().to_arrow()

# Spark also encrypts the Parquet data files, so they can't be read without their keys
table = _load_table(session_catalog_hive, identifier)
data_files = [task.file.file_path for task in table.scan().plan_files()]
assert data_files
for data_file in data_files:
with table.io.new_input(data_file).open() as stream:
with pytest.raises(OSError, match="no decryption found"):
pq.read_table(stream)


@pytest.mark.integration
@pytest.mark.parametrize("data_key_length", DATA_KEY_LENGTHS)
def test_inspect_encrypted_table(spark: SparkSession, session_catalog_hive: Catalog, data_key_length: int) -> None:
identifier = "default.test_inspect_encrypted_table"
_create_encrypted_table(spark, identifier, data_key_length)
spark.createDataFrame(ROWS).writeTo(f"hive.{identifier}").append()
# Delete a row from each partition, so Spark writes a deletion vector per partition
spark.sql(f"DELETE FROM hive.{identifier} WHERE id IN (1, 2)")
inspect = _load_table(session_catalog_hive, identifier).inspect

files = inspect.files()
spark_file_paths = [row.file_path for row in spark.table(f"hive.{identifier}.files").collect()]
assert sorted(files["file_path"].to_pylist()) == sorted(spark_file_paths)
assert all(key_metadata is not None for key_metadata in files["key_metadata"].to_pylist())
assert sorted(entry["data_file"]["file_path"] for entry in inspect.entries().to_pylist()) == sorted(spark_file_paths)

spark_manifest_paths = [row.path for row in spark.table(f"hive.{identifier}.manifests").collect()]
assert sorted(inspect.manifests()["path"].to_pylist()) == sorted(spark_manifest_paths)

# Each partition has two rows, and a deletion vector for one of them
partitions = [
(row["partition"]["category"], row["record_count"], row["position_delete_record_count"])
for row in inspect.partitions().to_pylist()
]
assert sorted(partitions) == [("a", 2, 1), ("b", 2, 1)]


def _create_encrypted_table(spark: SparkSession, identifier: str, data_key_length: int) -> None:
# As of Iceberg Java 1.12, only the hive catalog supports table encryption.
# Spark's hive catalog uses the mock KMS configured in dev/spark/spark-defaults.conf.
kms = boto3.client(
"kms",
endpoint_url="http://localhost:5001",
region_name="us-east-1",
aws_access_key_id="admin",
aws_secret_access_key="password",
)
master_key_id = kms.create_key()["KeyMetadata"]["KeyId"]

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

creates a new master_key_id per table


spark.sql(f"DROP TABLE IF EXISTS hive.{identifier}")
spark.sql(
f"""
CREATE TABLE hive.{identifier} (id bigint, category string, payload string)
USING iceberg
PARTITIONED BY (category)
TBLPROPERTIES (
'format-version'='3',
'encryption.key-id'='{master_key_id}',
'encryption.data-key-length'='{data_key_length}',
'write.delete.mode'='merge-on-read'
)
"""
)


def _load_table(catalog: Catalog, identifier: str) -> Table:
table = catalog.load_table(identifier)
return Table(
identifier=table.name(),
metadata=table.metadata,
metadata_location=table.metadata_location,
io=table.io,
catalog=catalog,
kms_client=AwsKeyManagementClient(
{
KMS_ENDPOINT: "http://localhost:5001",
AWS_REGION: "us-east-1",
AWS_ACCESS_KEY_ID: "admin",
AWS_SECRET_ACCESS_KEY: "password",
}
),
)
Loading