From dca6dfc4fd55408d9fa216ee6068a766af916711 Mon Sep 17 00:00:00 2001 From: Om Singhal Date: Mon, 28 Sep 2026 19:48:01 -0400 Subject: [PATCH 1/2] fix(idempotency): serialize UUID and datetime fields in dataclass payloads #8075 made _prepare_data dump Pydantic models with mode="json", but the dataclass branch still returned the raw dataclasses.asdict() output. A dataclass holding a UUID, date, time, datetime, plain Enum or path failed on every call with "Object of type UUID is not JSON serializable" while hashing the idempotency key. Convert those values to JSON safe ones after asdict(). Values json already encodes, including str and int enums, are left as they are, so keys for dataclasses that worked before don't change. --- .../utilities/idempotency/base.py | 31 ++++++++++- .../idempotency/_boto3/test_idempotency.py | 55 +++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/idempotency/base.py b/aws_lambda_powertools/utilities/idempotency/base.py index bee109ef842..e66d791af33 100644 --- a/aws_lambda_powertools/utilities/idempotency/base.py +++ b/aws_lambda_powertools/utilities/idempotency/base.py @@ -9,7 +9,10 @@ import datetime import logging from copy import deepcopy +from enum import Enum +from pathlib import PurePath from typing import TYPE_CHECKING, Any +from uuid import UUID from aws_lambda_powertools.utilities.idempotency.exceptions import ( IdempotencyAlreadyInProgressError, @@ -45,6 +48,32 @@ logger = logging.getLogger(__name__) +def _to_json_safe(value: Any) -> Any: + """Convert UUID, date, datetime, time, Enum and path values in a dataclass dict to JSON safe values. + + Values json already encodes are returned untouched, so idempotency keys that worked before don't change. + """ + if isinstance(value, dict): + return {key: _to_json_safe(item) for key, item in value.items()} + if isinstance(value, list): + return list(map(_to_json_safe, value)) + # tuples stay tuples so jmespath results don't change + if isinstance(value, tuple): + return tuple(map(_to_json_safe, value)) + return _json_safe_scalar(value) + + +def _json_safe_scalar(value: Any) -> Any: + # json already encodes these, including str and int based enums + if isinstance(value, (str, int, float)): + return value + if isinstance(value, Enum): + return _to_json_safe(value.value) + if isinstance(value, (datetime.date, datetime.time)): + return value.isoformat() + return str(value) if isinstance(value, (UUID, PurePath)) else value + + def _prepare_data(data: Any) -> Any: """Prepare data for json serialization. @@ -56,7 +85,7 @@ def _prepare_data(data: Any) -> Any: if hasattr(data, "__dataclass_fields__"): import dataclasses - return dataclasses.asdict(data) + return _to_json_safe(dataclasses.asdict(data)) # Convert from Pydantic model if callable(getattr(data, "model_dump", None)): diff --git a/tests/functional/idempotency/_boto3/test_idempotency.py b/tests/functional/idempotency/_boto3/test_idempotency.py index a4fb8db9132..b84588fc54f 100644 --- a/tests/functional/idempotency/_boto3/test_idempotency.py +++ b/tests/functional/idempotency/_boto3/test_idempotency.py @@ -2,8 +2,12 @@ import dataclasses import datetime import warnings +from decimal import Decimal +from enum import Enum +from pathlib import PurePosixPath from typing import Any from unittest.mock import MagicMock, Mock +from uuid import UUID import jmespath import pytest @@ -1772,6 +1776,57 @@ def collect_payment(payment: Payment): assert result == payment.transaction_id +def test_idempotent_function_dataclass_with_non_json_types(): + # GIVEN a dataclass with UUID, datetime, date, time, Enum and path values, some inside a list or tuple + config = IdempotencyConfig(use_local_cache=True) + mock_event = { + "order_id": "12345678-1234-5678-1234-567812345678", + "created_at": "2024-03-20T14:30:00", + "ship_date": "2024-03-22", + "delivery_window": ["09:00:00", "12:00:00"], + "item_ids": ["87654321-4321-8765-4321-876543218765"], + "status": "paid", + "receipt": "/receipts/ord-001.pdf", + "amount": "199.99", + } + idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_dataclass_with_non_json_types..process_order#{hash_idempotency_key(mock_event)}" # noqa E501 + persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key) + + class OrderStatus(Enum): + PAID = "paid" + + @dataclasses.dataclass + class Order: + order_id: UUID + created_at: datetime.datetime + ship_date: datetime.date + delivery_window: tuple[datetime.time, datetime.time] + item_ids: list[UUID] + status: OrderStatus + receipt: PurePosixPath + amount: Decimal + + @idempotent_function(data_keyword_argument="order", persistence_store=persistence_layer, config=config) + def process_order(order: Order) -> dict: + return {"status": "ok"} + + # WHEN + order = Order( + order_id=UUID("12345678-1234-5678-1234-567812345678"), + created_at=datetime.datetime(2024, 3, 20, 14, 30, 0), + ship_date=datetime.date(2024, 3, 22), + delivery_window=(datetime.time(9, 0), datetime.time(12, 0)), + item_ids=[UUID("87654321-4321-8765-4321-876543218765")], + status=OrderStatus.PAID, + receipt=PurePosixPath("/receipts/ord-001.pdf"), + amount=Decimal("199.99"), + ) + result = process_order(order=order) + + # THEN the key is hashed from the JSON version of the order, asserted at MockPersistenceLayer + assert result == {"status": "ok"} + + @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}], indirect=True) def test_idempotent_lambda_compound_already_completed( idempotency_config: IdempotencyConfig, From 8686fc62f5f05dece1548088d46eb0403cfa547c Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Tue, 29 Sep 2026 01:06:32 +0000 Subject: [PATCH 2/2] fix(idempotency): preserve existing hashes when encoding input types --- .../utilities/idempotency/base.py | 31 +-- .../utilities/idempotency/persistence/base.py | 22 +- docs/utilities/idempotency.md | 4 + .../idempotency/_boto3/test_idempotency.py | 9 +- .../test_idempotency_hash_serialization.py | 256 ++++++++++++++++++ 5 files changed, 289 insertions(+), 33 deletions(-) create mode 100644 tests/functional/idempotency/_boto3/test_idempotency_hash_serialization.py diff --git a/aws_lambda_powertools/utilities/idempotency/base.py b/aws_lambda_powertools/utilities/idempotency/base.py index e66d791af33..bee109ef842 100644 --- a/aws_lambda_powertools/utilities/idempotency/base.py +++ b/aws_lambda_powertools/utilities/idempotency/base.py @@ -9,10 +9,7 @@ import datetime import logging from copy import deepcopy -from enum import Enum -from pathlib import PurePath from typing import TYPE_CHECKING, Any -from uuid import UUID from aws_lambda_powertools.utilities.idempotency.exceptions import ( IdempotencyAlreadyInProgressError, @@ -48,32 +45,6 @@ logger = logging.getLogger(__name__) -def _to_json_safe(value: Any) -> Any: - """Convert UUID, date, datetime, time, Enum and path values in a dataclass dict to JSON safe values. - - Values json already encodes are returned untouched, so idempotency keys that worked before don't change. - """ - if isinstance(value, dict): - return {key: _to_json_safe(item) for key, item in value.items()} - if isinstance(value, list): - return list(map(_to_json_safe, value)) - # tuples stay tuples so jmespath results don't change - if isinstance(value, tuple): - return tuple(map(_to_json_safe, value)) - return _json_safe_scalar(value) - - -def _json_safe_scalar(value: Any) -> Any: - # json already encodes these, including str and int based enums - if isinstance(value, (str, int, float)): - return value - if isinstance(value, Enum): - return _to_json_safe(value.value) - if isinstance(value, (datetime.date, datetime.time)): - return value.isoformat() - return str(value) if isinstance(value, (UUID, PurePath)) else value - - def _prepare_data(data: Any) -> Any: """Prepare data for json serialization. @@ -85,7 +56,7 @@ def _prepare_data(data: Any) -> Any: if hasattr(data, "__dataclass_fields__"): import dataclasses - return _to_json_safe(dataclasses.asdict(data)) + return dataclasses.asdict(data) # Convert from Pydantic model if callable(getattr(data, "model_dump", None)): diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/base.py b/aws_lambda_powertools/utilities/idempotency/persistence/base.py index 739ba95dc23..ef3cff183a0 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/base.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/base.py @@ -11,7 +11,10 @@ import os import warnings from abc import ABC, abstractmethod +from enum import Enum +from pathlib import PurePath from typing import TYPE_CHECKING, Any +from uuid import UUID import jmespath @@ -35,6 +38,23 @@ logger = logging.getLogger(__name__) +class _IdempotencyKeyEncoder(Encoder): + """Extend hash serialization while preserving values the existing encoder supports.""" + + def default(self, obj: Any) -> Any: + try: + # Preserve existing encodings, including Decimal subclasses mixed with Enum. + return super().default(obj) + except TypeError: + if isinstance(obj, Enum): + return obj.value + if isinstance(obj, (datetime.date, datetime.time)): + return obj.isoformat() + if isinstance(obj, (UUID, PurePath)): + return str(obj) + raise + + class BasePersistenceLayer(ABC): """ Abstract Base Class for Idempotency persistence layer. @@ -178,7 +198,7 @@ def _generate_hash(self, data: Any) -> str: Hashed representation of the provided data """ - hashed_data = self.hash_function(json.dumps(data, cls=Encoder, sort_keys=True).encode()) + hashed_data = self.hash_function(json.dumps(data, cls=_IdempotencyKeyEncoder, sort_keys=True).encode()) return hashed_data.hexdigest() def _validate_payload( diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 06797b0563b..74f72bce585 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -192,6 +192,10 @@ When using this decorator, you **must** call your decorated function using keywo You can use `data_keyword_argument` to tell us the argument to extract an idempotency key. We support JSON serializable data, [Dataclasses](https://docs.python.org/3.12/library/dataclasses.html){target="_blank" rel="nofollow"}, Pydantic Models, and [Event Source Data Classes](./data_classes.md){target="_blank"} +When hashing dataclass or dictionary inputs, UUID and path values are encoded as strings, `datetime`, `date`, and `time` values use ISO 8601 strings, and enums use their values unless the existing encoder already supports the enum type. Existing encodings, including `Decimal` and enums based on strings, integers, floats, or `Decimal`, are preserved. + +These conversions happen after evaluating `event_key_jmespath` and `payload_validation_jmespath`, so their expressions and custom functions receive the original field types. They apply to key and validation hashing; function results continue to use the [output serialization](#output-serialization) configuration. + === "Using Dataclasses" ```python title="working_with_idempotent_function_dataclass.py" hl_lines="4-8 12 28 41" diff --git a/tests/functional/idempotency/_boto3/test_idempotency.py b/tests/functional/idempotency/_boto3/test_idempotency.py index b84588fc54f..74ece5c5bab 100644 --- a/tests/functional/idempotency/_boto3/test_idempotency.py +++ b/tests/functional/idempotency/_boto3/test_idempotency.py @@ -1776,9 +1776,9 @@ def collect_payment(payment: Payment): assert result == payment.transaction_id -def test_idempotent_function_dataclass_with_non_json_types(): +def test_idempotent_function_dataclass_with_non_json_types(lambda_context): # GIVEN a dataclass with UUID, datetime, date, time, Enum and path values, some inside a list or tuple - config = IdempotencyConfig(use_local_cache=True) + config = IdempotencyConfig(use_local_cache=True, lambda_context=lambda_context) mock_event = { "order_id": "12345678-1234-5678-1234-567812345678", "created_at": "2024-03-20T14:30:00", @@ -1806,8 +1806,11 @@ class Order: receipt: PurePosixPath amount: Decimal + executions = [] + @idempotent_function(data_keyword_argument="order", persistence_store=persistence_layer, config=config) def process_order(order: Order) -> dict: + executions.append(order) return {"status": "ok"} # WHEN @@ -1825,6 +1828,8 @@ def process_order(order: Order) -> dict: # THEN the key is hashed from the JSON version of the order, asserted at MockPersistenceLayer assert result == {"status": "ok"} + assert process_order(order=order) == result + assert executions == [order] @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}], indirect=True) diff --git a/tests/functional/idempotency/_boto3/test_idempotency_hash_serialization.py b/tests/functional/idempotency/_boto3/test_idempotency_hash_serialization.py new file mode 100644 index 00000000000..1eb81d11f78 --- /dev/null +++ b/tests/functional/idempotency/_boto3/test_idempotency_hash_serialization.py @@ -0,0 +1,256 @@ +import dataclasses +import datetime +from decimal import Decimal +from enum import Enum, IntEnum +from pathlib import PurePosixPath +from typing import Any, NamedTuple + +import pytest +from botocore import stub +from jmespath import functions + +from aws_lambda_powertools.utilities.idempotency import IdempotencyConfig, idempotent_function +from aws_lambda_powertools.utilities.idempotency.exceptions import ( + IdempotencyPersistenceLayerError, + IdempotencyValidationError, +) +from tests.functional.idempotency.utils import ( + build_idempotency_put_item_stub, + build_idempotency_update_item_stub, + hash_idempotency_key, +) +from tests.functional.utils import json_serialize + + +@dataclasses.dataclass +class Payload: + order_id: str + value: Any + + +class DecimalAmount(Decimal, Enum): + SMALL = Decimal("1.00") + + +class StringStatus(str, Enum): + PAID = "paid" + + +class IntegerStatus(IntEnum): + PAID = 1 + + +class FloatStatus(float, Enum): + PAID = 1.5 + + +class Point(NamedTuple): + x: int + y: int + + +def _expected_put(key_data, validation_data): + params = build_idempotency_put_item_stub(key_data) + params["Item"]["id"] = {"S": f"orders#{hash_idempotency_key(key_data)}"} + params["Item"]["validation"] = {"S": hash_idempotency_key(validation_data)} + return params + + +def _completed_record(key_data, validation_data, expiration): + return { + "Item": { + "id": {"S": f"orders#{hash_idempotency_key(key_data)}"}, + "expiration": {"N": expiration}, + "status": {"S": "COMPLETED"}, + "data": {"S": json_serialize({"status": "already processed"})}, + "validation": {"S": hash_idempotency_key(validation_data)}, + }, + } + + +def _assert_existing_record_is_reused(store, config, payload, key_data, validation_data, expiration): + @idempotent_function( + data_keyword_argument="payload", + persistence_store=store, + config=config, + key_prefix="orders", + ) + def process(payload): + pytest.fail("An existing record must be reused without executing the function") + + # Both hashes are produced by the existing shared Encoder, independently of the new hash encoder. + with stub.Stubber(store.client) as stubber: + stubber.add_client_error( + "put_item", + "ConditionalCheckFailedException", + modeled_fields=_completed_record(key_data, validation_data, expiration), + expected_params=_expected_put(key_data, validation_data), + ) + assert process(payload=payload) == {"status": "already processed"} + stubber.assert_no_pending_responses() + + +@pytest.mark.parametrize("as_dataclass", [True, False], ids=["dataclass", "dict"]) +@pytest.mark.parametrize( + "value", + [ + pytest.param({"text": "café", "integer": 1, "float": 1.5, "bool": True, "null": None}, id="primitives"), + pytest.param(Decimal("1.00"), id="decimal"), + pytest.param(Decimal("NaN"), id="decimal-nan"), + pytest.param(DecimalAmount.SMALL, id="decimal-enum"), + pytest.param(StringStatus.PAID, id="str-enum"), + pytest.param(IntegerStatus.PAID, id="int-enum"), + pytest.param(FloatStatus.PAID, id="float-enum"), + pytest.param((1, "two", Decimal("3.00")), id="tuple"), + pytest.param(Point(1, 2), id="namedtuple"), + pytest.param(Payload("nested", Decimal("1.00")), id="nested-dataclass"), + ], +) +def test_existing_key_and_validation_hashes_are_preserved( + persistence_store, + lambda_context, + timestamp_future, + as_dataclass, + value, +): + payload = Payload("order-1", value) + legacy_data = dataclasses.asdict(payload) + config = IdempotencyConfig(payload_validation_jmespath="value", lambda_context=lambda_context) + + _assert_existing_record_is_reused( + persistence_store, + config, + payload if as_dataclass else legacy_data, + legacy_data, + legacy_data["value"], + timestamp_future, + ) + + +@pytest.mark.parametrize( + "value,expected_type", + [ + pytest.param(datetime.datetime(2024, 3, 20, 14, 30), None, id="datetime"), + pytest.param(DecimalAmount.SMALL, None, id="decimal-enum"), + pytest.param(Point(1, 2), None, id="namedtuple"), + pytest.param([1, 2], "array", id="list"), + ], +) +@pytest.mark.parametrize("custom_function", [False, True], ids=["builtin", "custom"]) +def test_jmespath_sees_original_types_before_hashing( + persistence_store, + lambda_context, + timestamp_future, + value, + expected_type, + custom_function, +): + class CustomFunctions(functions.Functions): + @functions.signature({"types": []}) + def _func_original_type(self, value): + return type(value).__name__ + + if custom_function: + expression = "original_type(value)" + expected_type = type(value).__name__ + else: + expression = "type(value)" + + key_data = {"order": "order-1", "kind": expected_type} + validation_data = {"kind": expected_type} + config = IdempotencyConfig( + event_key_jmespath=f"{{order: order_id, kind: {expression}}}", + payload_validation_jmespath=f"{{kind: {expression}}}", + jmespath_options={"custom_functions": CustomFunctions()}, + lambda_context=lambda_context, + ) + _assert_existing_record_is_reused( + persistence_store, + config, + Payload("order-1", value), + key_data, + validation_data, + timestamp_future, + ) + + +@pytest.mark.parametrize("as_dataclass", [True, False], ids=["dataclass", "dict"]) +def test_new_types_support_persistent_replay_and_payload_validation( + persistence_store, + lambda_context, + timestamp_future, + as_dataclass, +): + value = datetime.datetime(2024, 3, 20, 14, 30, tzinfo=datetime.timezone.utc) + payload = Payload("order-1", {"created_at": value, "receipt": PurePosixPath("/orders/receipt.pdf")}) + prepared_value = {"created_at": value.isoformat(), "receipt": "/orders/receipt.pdf"} + config = IdempotencyConfig( + event_key_jmespath="order_id", + payload_validation_jmespath="value", + lambda_context=lambda_context, + use_local_cache=False, + ) + executions = [] + + @idempotent_function( + data_keyword_argument="payload", + persistence_store=persistence_store, + config=config, + key_prefix="orders", + ) + def process(payload): + executions.append(payload) + return {"status": "already processed"} + + expected_update = build_idempotency_update_item_stub("order-1", {"status": "already processed"}) + expected_update["Key"]["id"] = {"S": f"orders#{hash_idempotency_key('order-1')}"} + expected_update["ExpressionAttributeNames"]["#validation_key"] = "validation" + expected_update["ExpressionAttributeValues"][":validation_key"] = {"S": hash_idempotency_key(prepared_value)} + expected_update["UpdateExpression"] += ", #validation_key = :validation_key" + + with stub.Stubber(persistence_store.client) as stubber: + stubber.add_response("put_item", {}, _expected_put("order-1", prepared_value)) + stubber.add_response("update_item", {}, expected_update) + stubber.add_client_error( + "put_item", + "ConditionalCheckFailedException", + modeled_fields=_completed_record("order-1", prepared_value, timestamp_future), + expected_params=_expected_put("order-1", prepared_value), + ) + + request = payload if as_dataclass else dataclasses.asdict(payload) + assert process(payload=request) == {"status": "already processed"} + assert process(payload=request) == {"status": "already processed"} + assert len(executions) == 1 + + changed = dataclasses.replace(payload, value={**payload.value, "receipt": PurePosixPath("/other.pdf")}) + changed_value = {**prepared_value, "receipt": "/other.pdf"} + stubber.add_client_error( + "put_item", + "ConditionalCheckFailedException", + modeled_fields=_completed_record("order-1", prepared_value, timestamp_future), + expected_params=_expected_put("order-1", changed_value), + ) + with pytest.raises(IdempotencyValidationError): + process(payload=changed if as_dataclass else dataclasses.asdict(changed)) + + assert len(executions) == 1 + stubber.assert_no_pending_responses() + + +@pytest.mark.parametrize("value", [b"bytes", {1, 2}, complex(1, 2)], ids=["bytes", "set", "complex"]) +def test_unsupported_key_types_still_raise(persistence_store, lambda_context, value): + @idempotent_function( + data_keyword_argument="payload", + persistence_store=persistence_store, + config=IdempotencyConfig(lambda_context=lambda_context), + ) + def process(payload): + pytest.fail("Unsupported key types must fail before executing the function") + + with stub.Stubber(persistence_store.client) as stubber: + with pytest.raises(IdempotencyPersistenceLayerError) as error: + process(payload=Payload("order-1", value)) + assert isinstance(error.value.__cause__, TypeError) + assert "is not JSON serializable" in str(error.value.__cause__) + stubber.assert_no_pending_responses()