diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/base.py b/aws_lambda_powertools/utilities/idempotency/persistence/base.py index 739ba95dc23..ef3cff183a0 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/base.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/base.py @@ -11,7 +11,10 @@ import os import warnings from abc import ABC, abstractmethod +from enum import Enum +from pathlib import PurePath from typing import TYPE_CHECKING, Any +from uuid import UUID import jmespath @@ -35,6 +38,23 @@ logger = logging.getLogger(__name__) +class _IdempotencyKeyEncoder(Encoder): + """Extend hash serialization while preserving values the existing encoder supports.""" + + def default(self, obj: Any) -> Any: + try: + # Preserve existing encodings, including Decimal subclasses mixed with Enum. + return super().default(obj) + except TypeError: + if isinstance(obj, Enum): + return obj.value + if isinstance(obj, (datetime.date, datetime.time)): + return obj.isoformat() + if isinstance(obj, (UUID, PurePath)): + return str(obj) + raise + + class BasePersistenceLayer(ABC): """ Abstract Base Class for Idempotency persistence layer. @@ -178,7 +198,7 @@ def _generate_hash(self, data: Any) -> str: Hashed representation of the provided data """ - hashed_data = self.hash_function(json.dumps(data, cls=Encoder, sort_keys=True).encode()) + hashed_data = self.hash_function(json.dumps(data, cls=_IdempotencyKeyEncoder, sort_keys=True).encode()) return hashed_data.hexdigest() def _validate_payload( diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 06797b0563b..74f72bce585 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -192,6 +192,10 @@ When using this decorator, you **must** call your decorated function using keywo You can use `data_keyword_argument` to tell us the argument to extract an idempotency key. We support JSON serializable data, [Dataclasses](https://docs.python.org/3.12/library/dataclasses.html){target="_blank" rel="nofollow"}, Pydantic Models, and [Event Source Data Classes](./data_classes.md){target="_blank"} +When hashing dataclass or dictionary inputs, UUID and path values are encoded as strings, `datetime`, `date`, and `time` values use ISO 8601 strings, and enums use their values unless the existing encoder already supports the enum type. Existing encodings, including `Decimal` and enums based on strings, integers, floats, or `Decimal`, are preserved. + +These conversions happen after evaluating `event_key_jmespath` and `payload_validation_jmespath`, so their expressions and custom functions receive the original field types. They apply to key and validation hashing; function results continue to use the [output serialization](#output-serialization) configuration. + === "Using Dataclasses" ```python title="working_with_idempotent_function_dataclass.py" hl_lines="4-8 12 28 41" diff --git a/tests/functional/idempotency/_boto3/test_idempotency.py b/tests/functional/idempotency/_boto3/test_idempotency.py index a4fb8db9132..74ece5c5bab 100644 --- a/tests/functional/idempotency/_boto3/test_idempotency.py +++ b/tests/functional/idempotency/_boto3/test_idempotency.py @@ -2,8 +2,12 @@ import dataclasses import datetime import warnings +from decimal import Decimal +from enum import Enum +from pathlib import PurePosixPath from typing import Any from unittest.mock import MagicMock, Mock +from uuid import UUID import jmespath import pytest @@ -1772,6 +1776,62 @@ def collect_payment(payment: Payment): assert result == payment.transaction_id +def test_idempotent_function_dataclass_with_non_json_types(lambda_context): + # GIVEN a dataclass with UUID, datetime, date, time, Enum and path values, some inside a list or tuple + config = IdempotencyConfig(use_local_cache=True, lambda_context=lambda_context) + mock_event = { + "order_id": "12345678-1234-5678-1234-567812345678", + "created_at": "2024-03-20T14:30:00", + "ship_date": "2024-03-22", + "delivery_window": ["09:00:00", "12:00:00"], + "item_ids": ["87654321-4321-8765-4321-876543218765"], + "status": "paid", + "receipt": "/receipts/ord-001.pdf", + "amount": "199.99", + } + idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_dataclass_with_non_json_types..process_order#{hash_idempotency_key(mock_event)}" # noqa E501 + persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key) + + class OrderStatus(Enum): + PAID = "paid" + + @dataclasses.dataclass + class Order: + order_id: UUID + created_at: datetime.datetime + ship_date: datetime.date + delivery_window: tuple[datetime.time, datetime.time] + item_ids: list[UUID] + status: OrderStatus + receipt: PurePosixPath + amount: Decimal + + executions = [] + + @idempotent_function(data_keyword_argument="order", persistence_store=persistence_layer, config=config) + def process_order(order: Order) -> dict: + executions.append(order) + return {"status": "ok"} + + # WHEN + order = Order( + order_id=UUID("12345678-1234-5678-1234-567812345678"), + created_at=datetime.datetime(2024, 3, 20, 14, 30, 0), + ship_date=datetime.date(2024, 3, 22), + delivery_window=(datetime.time(9, 0), datetime.time(12, 0)), + item_ids=[UUID("87654321-4321-8765-4321-876543218765")], + status=OrderStatus.PAID, + receipt=PurePosixPath("/receipts/ord-001.pdf"), + amount=Decimal("199.99"), + ) + result = process_order(order=order) + + # THEN the key is hashed from the JSON version of the order, asserted at MockPersistenceLayer + assert result == {"status": "ok"} + assert process_order(order=order) == result + assert executions == [order] + + @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}], indirect=True) def test_idempotent_lambda_compound_already_completed( idempotency_config: IdempotencyConfig, diff --git a/tests/functional/idempotency/_boto3/test_idempotency_hash_serialization.py b/tests/functional/idempotency/_boto3/test_idempotency_hash_serialization.py new file mode 100644 index 00000000000..1eb81d11f78 --- /dev/null +++ b/tests/functional/idempotency/_boto3/test_idempotency_hash_serialization.py @@ -0,0 +1,256 @@ +import dataclasses +import datetime +from decimal import Decimal +from enum import Enum, IntEnum +from pathlib import PurePosixPath +from typing import Any, NamedTuple + +import pytest +from botocore import stub +from jmespath import functions + +from aws_lambda_powertools.utilities.idempotency import IdempotencyConfig, idempotent_function +from aws_lambda_powertools.utilities.idempotency.exceptions import ( + IdempotencyPersistenceLayerError, + IdempotencyValidationError, +) +from tests.functional.idempotency.utils import ( + build_idempotency_put_item_stub, + build_idempotency_update_item_stub, + hash_idempotency_key, +) +from tests.functional.utils import json_serialize + + +@dataclasses.dataclass +class Payload: + order_id: str + value: Any + + +class DecimalAmount(Decimal, Enum): + SMALL = Decimal("1.00") + + +class StringStatus(str, Enum): + PAID = "paid" + + +class IntegerStatus(IntEnum): + PAID = 1 + + +class FloatStatus(float, Enum): + PAID = 1.5 + + +class Point(NamedTuple): + x: int + y: int + + +def _expected_put(key_data, validation_data): + params = build_idempotency_put_item_stub(key_data) + params["Item"]["id"] = {"S": f"orders#{hash_idempotency_key(key_data)}"} + params["Item"]["validation"] = {"S": hash_idempotency_key(validation_data)} + return params + + +def _completed_record(key_data, validation_data, expiration): + return { + "Item": { + "id": {"S": f"orders#{hash_idempotency_key(key_data)}"}, + "expiration": {"N": expiration}, + "status": {"S": "COMPLETED"}, + "data": {"S": json_serialize({"status": "already processed"})}, + "validation": {"S": hash_idempotency_key(validation_data)}, + }, + } + + +def _assert_existing_record_is_reused(store, config, payload, key_data, validation_data, expiration): + @idempotent_function( + data_keyword_argument="payload", + persistence_store=store, + config=config, + key_prefix="orders", + ) + def process(payload): + pytest.fail("An existing record must be reused without executing the function") + + # Both hashes are produced by the existing shared Encoder, independently of the new hash encoder. + with stub.Stubber(store.client) as stubber: + stubber.add_client_error( + "put_item", + "ConditionalCheckFailedException", + modeled_fields=_completed_record(key_data, validation_data, expiration), + expected_params=_expected_put(key_data, validation_data), + ) + assert process(payload=payload) == {"status": "already processed"} + stubber.assert_no_pending_responses() + + +@pytest.mark.parametrize("as_dataclass", [True, False], ids=["dataclass", "dict"]) +@pytest.mark.parametrize( + "value", + [ + pytest.param({"text": "café", "integer": 1, "float": 1.5, "bool": True, "null": None}, id="primitives"), + pytest.param(Decimal("1.00"), id="decimal"), + pytest.param(Decimal("NaN"), id="decimal-nan"), + pytest.param(DecimalAmount.SMALL, id="decimal-enum"), + pytest.param(StringStatus.PAID, id="str-enum"), + pytest.param(IntegerStatus.PAID, id="int-enum"), + pytest.param(FloatStatus.PAID, id="float-enum"), + pytest.param((1, "two", Decimal("3.00")), id="tuple"), + pytest.param(Point(1, 2), id="namedtuple"), + pytest.param(Payload("nested", Decimal("1.00")), id="nested-dataclass"), + ], +) +def test_existing_key_and_validation_hashes_are_preserved( + persistence_store, + lambda_context, + timestamp_future, + as_dataclass, + value, +): + payload = Payload("order-1", value) + legacy_data = dataclasses.asdict(payload) + config = IdempotencyConfig(payload_validation_jmespath="value", lambda_context=lambda_context) + + _assert_existing_record_is_reused( + persistence_store, + config, + payload if as_dataclass else legacy_data, + legacy_data, + legacy_data["value"], + timestamp_future, + ) + + +@pytest.mark.parametrize( + "value,expected_type", + [ + pytest.param(datetime.datetime(2024, 3, 20, 14, 30), None, id="datetime"), + pytest.param(DecimalAmount.SMALL, None, id="decimal-enum"), + pytest.param(Point(1, 2), None, id="namedtuple"), + pytest.param([1, 2], "array", id="list"), + ], +) +@pytest.mark.parametrize("custom_function", [False, True], ids=["builtin", "custom"]) +def test_jmespath_sees_original_types_before_hashing( + persistence_store, + lambda_context, + timestamp_future, + value, + expected_type, + custom_function, +): + class CustomFunctions(functions.Functions): + @functions.signature({"types": []}) + def _func_original_type(self, value): + return type(value).__name__ + + if custom_function: + expression = "original_type(value)" + expected_type = type(value).__name__ + else: + expression = "type(value)" + + key_data = {"order": "order-1", "kind": expected_type} + validation_data = {"kind": expected_type} + config = IdempotencyConfig( + event_key_jmespath=f"{{order: order_id, kind: {expression}}}", + payload_validation_jmespath=f"{{kind: {expression}}}", + jmespath_options={"custom_functions": CustomFunctions()}, + lambda_context=lambda_context, + ) + _assert_existing_record_is_reused( + persistence_store, + config, + Payload("order-1", value), + key_data, + validation_data, + timestamp_future, + ) + + +@pytest.mark.parametrize("as_dataclass", [True, False], ids=["dataclass", "dict"]) +def test_new_types_support_persistent_replay_and_payload_validation( + persistence_store, + lambda_context, + timestamp_future, + as_dataclass, +): + value = datetime.datetime(2024, 3, 20, 14, 30, tzinfo=datetime.timezone.utc) + payload = Payload("order-1", {"created_at": value, "receipt": PurePosixPath("/orders/receipt.pdf")}) + prepared_value = {"created_at": value.isoformat(), "receipt": "/orders/receipt.pdf"} + config = IdempotencyConfig( + event_key_jmespath="order_id", + payload_validation_jmespath="value", + lambda_context=lambda_context, + use_local_cache=False, + ) + executions = [] + + @idempotent_function( + data_keyword_argument="payload", + persistence_store=persistence_store, + config=config, + key_prefix="orders", + ) + def process(payload): + executions.append(payload) + return {"status": "already processed"} + + expected_update = build_idempotency_update_item_stub("order-1", {"status": "already processed"}) + expected_update["Key"]["id"] = {"S": f"orders#{hash_idempotency_key('order-1')}"} + expected_update["ExpressionAttributeNames"]["#validation_key"] = "validation" + expected_update["ExpressionAttributeValues"][":validation_key"] = {"S": hash_idempotency_key(prepared_value)} + expected_update["UpdateExpression"] += ", #validation_key = :validation_key" + + with stub.Stubber(persistence_store.client) as stubber: + stubber.add_response("put_item", {}, _expected_put("order-1", prepared_value)) + stubber.add_response("update_item", {}, expected_update) + stubber.add_client_error( + "put_item", + "ConditionalCheckFailedException", + modeled_fields=_completed_record("order-1", prepared_value, timestamp_future), + expected_params=_expected_put("order-1", prepared_value), + ) + + request = payload if as_dataclass else dataclasses.asdict(payload) + assert process(payload=request) == {"status": "already processed"} + assert process(payload=request) == {"status": "already processed"} + assert len(executions) == 1 + + changed = dataclasses.replace(payload, value={**payload.value, "receipt": PurePosixPath("/other.pdf")}) + changed_value = {**prepared_value, "receipt": "/other.pdf"} + stubber.add_client_error( + "put_item", + "ConditionalCheckFailedException", + modeled_fields=_completed_record("order-1", prepared_value, timestamp_future), + expected_params=_expected_put("order-1", changed_value), + ) + with pytest.raises(IdempotencyValidationError): + process(payload=changed if as_dataclass else dataclasses.asdict(changed)) + + assert len(executions) == 1 + stubber.assert_no_pending_responses() + + +@pytest.mark.parametrize("value", [b"bytes", {1, 2}, complex(1, 2)], ids=["bytes", "set", "complex"]) +def test_unsupported_key_types_still_raise(persistence_store, lambda_context, value): + @idempotent_function( + data_keyword_argument="payload", + persistence_store=persistence_store, + config=IdempotencyConfig(lambda_context=lambda_context), + ) + def process(payload): + pytest.fail("Unsupported key types must fail before executing the function") + + with stub.Stubber(persistence_store.client) as stubber: + with pytest.raises(IdempotencyPersistenceLayerError) as error: + process(payload=Payload("order-1", value)) + assert isinstance(error.value.__cause__, TypeError) + assert "is not JSON serializable" in str(error.value.__cause__) + stubber.assert_no_pending_responses()