fix(idempotency): serialize UUID and datetime fields in dataclass payloads

aws_lambda_powertools/utilities/idempotency/persistence/base.py

@@ -11,7 +11,10 @@
 import os
 import warnings
 from abc import ABC, abstractmethod
+from enum import Enum
+from pathlib import PurePath
 from typing import TYPE_CHECKING, Any
+from uuid import UUID
 
 import jmespath
 
@@ -35,6 +38,23 @@
 logger = logging.getLogger(__name__)
 
 
+class _IdempotencyKeyEncoder(Encoder):
+    """Extend hash serialization while preserving values the existing encoder supports."""
+
+    def default(self, obj: Any) -> Any:
+        try:
+            # Preserve existing encodings, including Decimal subclasses mixed with Enum.
+            return super().default(obj)
+        except TypeError:
+            if isinstance(obj, Enum):
+                return obj.value
+            if isinstance(obj, (datetime.date, datetime.time)):
+                return obj.isoformat()
+            if isinstance(obj, (UUID, PurePath)):
+                return str(obj)
+            raise
+
+
 class BasePersistenceLayer(ABC):
     """
     Abstract Base Class for Idempotency persistence layer.
@@ -178,7 +198,7 @@ def _generate_hash(self, data: Any) -> str:
             Hashed representation of the provided data
 
         """
-        hashed_data = self.hash_function(json.dumps(data, cls=Encoder, sort_keys=True).encode())
+        hashed_data = self.hash_function(json.dumps(data, cls=_IdempotencyKeyEncoder, sort_keys=True).encode())
         return hashed_data.hexdigest()
 
     def _validate_payload(

docs/utilities/idempotency.md

@@ -192,6 +192,10 @@ When using this decorator, you **must** call your decorated function using keywo
 
 You can use `data_keyword_argument` to tell us the argument to extract an idempotency key.  We support JSON serializable data, [Dataclasses](https://docs.python.org/3.12/library/dataclasses.html){target="_blank" rel="nofollow"}, Pydantic Models, and [Event Source Data Classes](./data_classes.md){target="_blank"}
 
+When hashing dataclass or dictionary inputs, UUID and path values are encoded as strings, `datetime`, `date`, and `time` values use ISO 8601 strings, and enums use their values unless the existing encoder already supports the enum type. Existing encodings, including `Decimal` and enums based on strings, integers, floats, or `Decimal`, are preserved.
+
+These conversions happen after evaluating `event_key_jmespath` and `payload_validation_jmespath`, so their expressions and custom functions receive the original field types. They apply to key and validation hashing; function results continue to use the [output serialization](#output-serialization) configuration.
+
 === "Using Dataclasses"
 
     ```python title="working_with_idempotent_function_dataclass.py" hl_lines="4-8 12 28 41"

tests/functional/idempotency/_boto3/test_idempotency.py

@@ -2,8 +2,12 @@
 import dataclasses
 import datetime
 import warnings
+from decimal import Decimal
+from enum import Enum
+from pathlib import PurePosixPath
 from typing import Any
 from unittest.mock import MagicMock, Mock
+from uuid import UUID
 
 import jmespath
 import pytest
@@ -1772,6 +1776,62 @@ def collect_payment(payment: Payment):
     assert result == payment.transaction_id
 
 
+def test_idempotent_function_dataclass_with_non_json_types(lambda_context):
+    # GIVEN a dataclass with UUID, datetime, date, time, Enum and path values, some inside a list or tuple
+    config = IdempotencyConfig(use_local_cache=True, lambda_context=lambda_context)
+    mock_event = {
+        "order_id": "12345678-1234-5678-1234-567812345678",
+        "created_at": "2024-03-20T14:30:00",
+        "ship_date": "2024-03-22",
+        "delivery_window": ["09:00:00", "12:00:00"],
+        "item_ids": ["87654321-4321-8765-4321-876543218765"],
+        "status": "paid",
+        "receipt": "/receipts/ord-001.pdf",
+        "amount": "199.99",
+    }
+    idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_dataclass_with_non_json_types.<locals>.process_order#{hash_idempotency_key(mock_event)}"  # noqa E501
+    persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key)
+
+    class OrderStatus(Enum):
+        PAID = "paid"
+
+    @dataclasses.dataclass
+    class Order:
+        order_id: UUID
+        created_at: datetime.datetime
+        ship_date: datetime.date
+        delivery_window: tuple[datetime.time, datetime.time]
+        item_ids: list[UUID]
+        status: OrderStatus
+        receipt: PurePosixPath
+        amount: Decimal
+
+    executions = []
+
+    @idempotent_function(data_keyword_argument="order", persistence_store=persistence_layer, config=config)
+    def process_order(order: Order) -> dict:
+        executions.append(order)
+        return {"status": "ok"}
+
+    # WHEN
+    order = Order(
+        order_id=UUID("12345678-1234-5678-1234-567812345678"),
+        created_at=datetime.datetime(2024, 3, 20, 14, 30, 0),
+        ship_date=datetime.date(2024, 3, 22),
+        delivery_window=(datetime.time(9, 0), datetime.time(12, 0)),
+        item_ids=[UUID("87654321-4321-8765-4321-876543218765")],
+        status=OrderStatus.PAID,
+        receipt=PurePosixPath("/receipts/ord-001.pdf"),
+        amount=Decimal("199.99"),
+    )
+    result = process_order(order=order)
+
+    # THEN the key is hashed from the JSON version of the order, asserted at MockPersistenceLayer
+    assert result == {"status": "ok"}
+    assert process_order(order=order) == result
+    assert executions == [order]
+
+
 @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}], indirect=True)
 def test_idempotent_lambda_compound_already_completed(
     idempotency_config: IdempotencyConfig,

tests/functional/idempotency/_boto3/test_idempotency_hash_serialization.py

@@ -0,0 +1,256 @@
+import dataclasses
+import datetime
+from decimal import Decimal
+from enum import Enum, IntEnum
+from pathlib import PurePosixPath
+from typing import Any, NamedTuple
+
+import pytest
+from botocore import stub
+from jmespath import functions
+
+from aws_lambda_powertools.utilities.idempotency import IdempotencyConfig, idempotent_function
+from aws_lambda_powertools.utilities.idempotency.exceptions import (
+    IdempotencyPersistenceLayerError,
+    IdempotencyValidationError,
+)
+from tests.functional.idempotency.utils import (
+    build_idempotency_put_item_stub,
+    build_idempotency_update_item_stub,
+    hash_idempotency_key,
+)
+from tests.functional.utils import json_serialize
+
+
+@dataclasses.dataclass
+class Payload:
+    order_id: str
+    value: Any
+
+
+class DecimalAmount(Decimal, Enum):
+    SMALL = Decimal("1.00")
+
+
+class StringStatus(str, Enum):
+    PAID = "paid"
+
+
+class IntegerStatus(IntEnum):
+    PAID = 1
+
+
+class FloatStatus(float, Enum):
+    PAID = 1.5
+
+
+class Point(NamedTuple):
+    x: int
+    y: int
+
+
+def _expected_put(key_data, validation_data):
+    params = build_idempotency_put_item_stub(key_data)
+    params["Item"]["id"] = {"S": f"orders#{hash_idempotency_key(key_data)}"}
+    params["Item"]["validation"] = {"S": hash_idempotency_key(validation_data)}
+    return params
+
+
+def _completed_record(key_data, validation_data, expiration):
+    return {
+        "Item": {
+            "id": {"S": f"orders#{hash_idempotency_key(key_data)}"},
+            "expiration": {"N": expiration},
+            "status": {"S": "COMPLETED"},
+            "data": {"S": json_serialize({"status": "already processed"})},
+            "validation": {"S": hash_idempotency_key(validation_data)},
+        },
+    }
+
+
+def _assert_existing_record_is_reused(store, config, payload, key_data, validation_data, expiration):
+    @idempotent_function(
+        data_keyword_argument="payload",
+        persistence_store=store,
+        config=config,
+        key_prefix="orders",
+    )
+    def process(payload):
+        pytest.fail("An existing record must be reused without executing the function")
+
+    # Both hashes are produced by the existing shared Encoder, independently of the new hash encoder.
+    with stub.Stubber(store.client) as stubber:
+        stubber.add_client_error(
+            "put_item",
+            "ConditionalCheckFailedException",
+            modeled_fields=_completed_record(key_data, validation_data, expiration),
+            expected_params=_expected_put(key_data, validation_data),
+        )
+        assert process(payload=payload) == {"status": "already processed"}
+        stubber.assert_no_pending_responses()
+
+
+@pytest.mark.parametrize("as_dataclass", [True, False], ids=["dataclass", "dict"])
+@pytest.mark.parametrize(
+    "value",
+    [
+        pytest.param({"text": "café", "integer": 1, "float": 1.5, "bool": True, "null": None}, id="primitives"),
+        pytest.param(Decimal("1.00"), id="decimal"),
+        pytest.param(Decimal("NaN"), id="decimal-nan"),
+        pytest.param(DecimalAmount.SMALL, id="decimal-enum"),
+        pytest.param(StringStatus.PAID, id="str-enum"),
+        pytest.param(IntegerStatus.PAID, id="int-enum"),
+        pytest.param(FloatStatus.PAID, id="float-enum"),
+        pytest.param((1, "two", Decimal("3.00")), id="tuple"),
+        pytest.param(Point(1, 2), id="namedtuple"),
+        pytest.param(Payload("nested", Decimal("1.00")), id="nested-dataclass"),
+    ],
+)
+def test_existing_key_and_validation_hashes_are_preserved(
+    persistence_store,
+    lambda_context,
+    timestamp_future,
+    as_dataclass,
+    value,
+):
+    payload = Payload("order-1", value)
+    legacy_data = dataclasses.asdict(payload)
+    config = IdempotencyConfig(payload_validation_jmespath="value", lambda_context=lambda_context)
+
+    _assert_existing_record_is_reused(
+        persistence_store,
+        config,
+        payload if as_dataclass else legacy_data,
+        legacy_data,
+        legacy_data["value"],
+        timestamp_future,
+    )
+
+
+@pytest.mark.parametrize(
+    "value,expected_type",
+    [
+        pytest.param(datetime.datetime(2024, 3, 20, 14, 30), None, id="datetime"),
+        pytest.param(DecimalAmount.SMALL, None, id="decimal-enum"),
+        pytest.param(Point(1, 2), None, id="namedtuple"),
+        pytest.param([1, 2], "array", id="list"),
+    ],
+)
+@pytest.mark.parametrize("custom_function", [False, True], ids=["builtin", "custom"])
+def test_jmespath_sees_original_types_before_hashing(
+    persistence_store,
+    lambda_context,
+    timestamp_future,
+    value,
+    expected_type,
+    custom_function,
+):
+    class CustomFunctions(functions.Functions):
+        @functions.signature({"types": []})
+        def _func_original_type(self, value):
+            return type(value).__name__
+
+    if custom_function:
+        expression = "original_type(value)"
+        expected_type = type(value).__name__
+    else:
+        expression = "type(value)"
+
+    key_data = {"order": "order-1", "kind": expected_type}
+    validation_data = {"kind": expected_type}
+    config = IdempotencyConfig(
+        event_key_jmespath=f"{{order: order_id, kind: {expression}}}",
+        payload_validation_jmespath=f"{{kind: {expression}}}",
+        jmespath_options={"custom_functions": CustomFunctions()},
+        lambda_context=lambda_context,
+    )
+    _assert_existing_record_is_reused(
+        persistence_store,
+        config,
+        Payload("order-1", value),
+        key_data,
+        validation_data,
+        timestamp_future,
+    )
+
+
+@pytest.mark.parametrize("as_dataclass", [True, False], ids=["dataclass", "dict"])
+def test_new_types_support_persistent_replay_and_payload_validation(
+    persistence_store,
+    lambda_context,
+    timestamp_future,
+    as_dataclass,
+):
+    value = datetime.datetime(2024, 3, 20, 14, 30, tzinfo=datetime.timezone.utc)
+    payload = Payload("order-1", {"created_at": value, "receipt": PurePosixPath("/orders/receipt.pdf")})
+    prepared_value = {"created_at": value.isoformat(), "receipt": "/orders/receipt.pdf"}
+    config = IdempotencyConfig(
+        event_key_jmespath="order_id",
+        payload_validation_jmespath="value",
+        lambda_context=lambda_context,
+        use_local_cache=False,
+    )
+    executions = []
+
+    @idempotent_function(
+        data_keyword_argument="payload",
+        persistence_store=persistence_store,
+        config=config,
+        key_prefix="orders",
+    )
+    def process(payload):
+        executions.append(payload)
+        return {"status": "already processed"}
+
+    expected_update = build_idempotency_update_item_stub("order-1", {"status": "already processed"})
+    expected_update["Key"]["id"] = {"S": f"orders#{hash_idempotency_key('order-1')}"}
+    expected_update["ExpressionAttributeNames"]["#validation_key"] = "validation"
+    expected_update["ExpressionAttributeValues"][":validation_key"] = {"S": hash_idempotency_key(prepared_value)}
+    expected_update["UpdateExpression"] += ", #validation_key = :validation_key"
+
+    with stub.Stubber(persistence_store.client) as stubber:
+        stubber.add_response("put_item", {}, _expected_put("order-1", prepared_value))
+        stubber.add_response("update_item", {}, expected_update)
+        stubber.add_client_error(
+            "put_item",
+            "ConditionalCheckFailedException",
+            modeled_fields=_completed_record("order-1", prepared_value, timestamp_future),
+            expected_params=_expected_put("order-1", prepared_value),
+        )
+
+        request = payload if as_dataclass else dataclasses.asdict(payload)
+        assert process(payload=request) == {"status": "already processed"}
+        assert process(payload=request) == {"status": "already processed"}
+        assert len(executions) == 1
+
+        changed = dataclasses.replace(payload, value={**payload.value, "receipt": PurePosixPath("/other.pdf")})
+        changed_value = {**prepared_value, "receipt": "/other.pdf"}
+        stubber.add_client_error(
+            "put_item",
+            "ConditionalCheckFailedException",
+            modeled_fields=_completed_record("order-1", prepared_value, timestamp_future),
+            expected_params=_expected_put("order-1", changed_value),
+        )
+        with pytest.raises(IdempotencyValidationError):
+            process(payload=changed if as_dataclass else dataclasses.asdict(changed))
+
+        assert len(executions) == 1
+        stubber.assert_no_pending_responses()
+
+
+@pytest.mark.parametrize("value", [b"bytes", {1, 2}, complex(1, 2)], ids=["bytes", "set", "complex"])
+def test_unsupported_key_types_still_raise(persistence_store, lambda_context, value):
+    @idempotent_function(
+        data_keyword_argument="payload",
+        persistence_store=persistence_store,
+        config=IdempotencyConfig(lambda_context=lambda_context),
+    )
+    def process(payload):
+        pytest.fail("Unsupported key types must fail before executing the function")
+
+    with stub.Stubber(persistence_store.client) as stubber:
+        with pytest.raises(IdempotencyPersistenceLayerError) as error:
+            process(payload=Payload("order-1", value))
+        assert isinstance(error.value.__cause__, TypeError)
+        assert "is not JSON serializable" in str(error.value.__cause__)
+        stubber.assert_no_pending_responses()