diff --git a/aws_lambda_powertools/utilities/parser/models/__init__.py b/aws_lambda_powertools/utilities/parser/models/__init__.py index ba18fd9f04a..2dc2985d4c5 100644 --- a/aws_lambda_powertools/utilities/parser/models/__init__.py +++ b/aws_lambda_powertools/utilities/parser/models/__init__.py @@ -1,136 +1,274 @@ -from .alb import AlbModel, AlbRequestContext, AlbRequestContextData -from .apigw import ( - ApiGatewayAuthorizerRequest, - ApiGatewayAuthorizerToken, - APIGatewayEventAuthorizer, - APIGatewayEventIdentity, - APIGatewayEventRequestContext, - APIGatewayProxyEventModel, -) -from .apigw_websocket import ( - APIGatewayWebSocketConnectEventModel, - APIGatewayWebSocketConnectEventRequestContext, - APIGatewayWebSocketDisconnectEventModel, - APIGatewayWebSocketDisconnectEventRequestContext, - APIGatewayWebSocketEventIdentity, - APIGatewayWebSocketEventRequestContextBase, - APIGatewayWebSocketMessageEventModel, - APIGatewayWebSocketMessageEventRequestContext, -) -from .apigwv2 import ( - ApiGatewayAuthorizerRequestV2, - APIGatewayProxyEventV2Model, - RequestContextV2, - RequestContextV2Authorizer, - RequestContextV2AuthorizerIam, - RequestContextV2AuthorizerIamCognito, - RequestContextV2AuthorizerJwt, - RequestContextV2Http, -) -from .appsync import AppSyncResolverEventModel -from .appsync_events import AppSyncEventsModel -from .bedrock_agent import ( - BedrockAgentEventModel, - BedrockAgentFunctionEventModel, - BedrockAgentModel, - BedrockAgentPropertyModel, - BedrockAgentRequestBodyModel, - BedrockAgentRequestMediaModel, -) -from .cloudformation_custom_resource import ( - CloudFormationCustomResourceBaseModel, - CloudFormationCustomResourceCreateModel, - CloudFormationCustomResourceDeleteModel, - CloudFormationCustomResourceUpdateModel, -) -from .cloudwatch import ( - CloudWatchLogsData, - CloudWatchLogsDecode, - CloudWatchLogsLogEvent, - CloudWatchLogsModel, -) -from .cognito import ( - CognitoCreateAuthChallengeTriggerModel, - CognitoCustomEmailSenderTriggerModel, - CognitoCustomMessageTriggerModel, - CognitoCustomSMSSenderTriggerModel, - CognitoDefineAuthChallengeTriggerModel, - CognitoMigrateUserTriggerModel, - CognitoPostAuthenticationTriggerModel, - CognitoPostConfirmationTriggerModel, - CognitoPreAuthenticationTriggerModel, - CognitoPreSignupTriggerModel, - CognitoPreTokenGenerationTriggerModelV1, - CognitoPreTokenGenerationTriggerModelV2AndV3, - CognitoVerifyAuthChallengeTriggerModel, -) -from .dynamodb import ( - DynamoDBStreamChangedRecordModel, - DynamoDBStreamLambdaOnFailureDestinationModel, - DynamoDBStreamModel, - DynamoDBStreamRecordModel, -) -from .event_bridge import EventBridgeModel -from .kafka import ( - KafkaBaseEventModel, - KafkaMskEventModel, - KafkaRecordModel, - KafkaSelfManagedEventModel, -) -from .kinesis import ( - KinesisDataStreamModel, - KinesisDataStreamRecord, - KinesisDataStreamRecordPayload, -) -from .kinesis_firehose import ( - KinesisFirehoseModel, - KinesisFirehoseRecord, - KinesisFirehoseRecordMetadata, -) -from .kinesis_firehose_sqs import KinesisFirehoseSqsModel, KinesisFirehoseSqsRecord -from .lambda_function_url import LambdaFunctionUrlModel -from .s3 import ( - S3EventNotificationEventBridgeDetailModel, - S3EventNotificationEventBridgeModel, - S3EventNotificationObjectModel, - S3Model, - S3RecordModel, -) -from .s3_batch_operation import ( - S3BatchOperationJobModel, - S3BatchOperationModel, - S3BatchOperationTaskModel, -) -from .s3_event_notification import ( - S3SqsEventNotificationModel, - S3SqsEventNotificationRecordModel, -) -from .s3_object_event import ( - S3ObjectConfiguration, - S3ObjectContext, - S3ObjectLambdaEvent, - S3ObjectSessionAttributes, - S3ObjectSessionContext, - S3ObjectSessionIssuer, - S3ObjectUserIdentity, - S3ObjectUserRequest, -) -from .ses import ( - SesMail, - SesMailCommonHeaders, - SesMailHeaders, - SesMessage, - SesModel, - SesReceipt, - SesReceiptAction, - SesReceiptVerdict, - SesRecordModel, -) -from .sns import SnsModel, SnsNotificationModel, SnsRecordModel -from .sqs import SqsAttributesModel, SqsModel, SqsMsgAttributeModel, SqsRecordModel -from .transfer_family import TransferFamilyAuthorizer -from .vpc_lattice import VpcLatticeModel -from .vpc_latticev2 import VpcLatticeV2Model +"""Event models loaded on first access to avoid initializing unrelated schemas.""" + +import importlib +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .alb import AlbModel, AlbRequestContext, AlbRequestContextData + from .apigw import ( + ApiGatewayAuthorizerRequest, + ApiGatewayAuthorizerToken, + APIGatewayEventAuthorizer, + APIGatewayEventIdentity, + APIGatewayEventRequestContext, + APIGatewayProxyEventModel, + ) + from .apigw_websocket import ( + APIGatewayWebSocketConnectEventModel, + APIGatewayWebSocketConnectEventRequestContext, + APIGatewayWebSocketDisconnectEventModel, + APIGatewayWebSocketDisconnectEventRequestContext, + APIGatewayWebSocketEventIdentity, + APIGatewayWebSocketEventRequestContextBase, + APIGatewayWebSocketMessageEventModel, + APIGatewayWebSocketMessageEventRequestContext, + ) + from .apigwv2 import ( + ApiGatewayAuthorizerRequestV2, + APIGatewayProxyEventV2Model, + RequestContextV2, + RequestContextV2Authorizer, + RequestContextV2AuthorizerIam, + RequestContextV2AuthorizerIamCognito, + RequestContextV2AuthorizerJwt, + RequestContextV2Http, + ) + from .appsync import AppSyncResolverEventModel + from .appsync_events import AppSyncEventsModel + from .bedrock_agent import ( + BedrockAgentEventModel, + BedrockAgentFunctionEventModel, + BedrockAgentModel, + BedrockAgentPropertyModel, + BedrockAgentRequestBodyModel, + BedrockAgentRequestMediaModel, + ) + from .cloudformation_custom_resource import ( + CloudFormationCustomResourceBaseModel, + CloudFormationCustomResourceCreateModel, + CloudFormationCustomResourceDeleteModel, + CloudFormationCustomResourceUpdateModel, + ) + from .cloudwatch import ( + CloudWatchLogsData, + CloudWatchLogsDecode, + CloudWatchLogsLogEvent, + CloudWatchLogsModel, + ) + from .cognito import ( + CognitoCreateAuthChallengeTriggerModel, + CognitoCustomEmailSenderTriggerModel, + CognitoCustomMessageTriggerModel, + CognitoCustomSMSSenderTriggerModel, + CognitoDefineAuthChallengeTriggerModel, + CognitoMigrateUserTriggerModel, + CognitoPostAuthenticationTriggerModel, + CognitoPostConfirmationTriggerModel, + CognitoPreAuthenticationTriggerModel, + CognitoPreSignupTriggerModel, + CognitoPreTokenGenerationTriggerModelV1, + CognitoPreTokenGenerationTriggerModelV2AndV3, + CognitoVerifyAuthChallengeTriggerModel, + ) + from .dynamodb import ( + DynamoDBStreamChangedRecordModel, + DynamoDBStreamLambdaOnFailureDestinationModel, + DynamoDBStreamModel, + DynamoDBStreamRecordModel, + ) + from .event_bridge import EventBridgeModel + from .kafka import ( + KafkaBaseEventModel, + KafkaMskEventModel, + KafkaRecordModel, + KafkaSelfManagedEventModel, + ) + from .kinesis import ( + KinesisDataStreamModel, + KinesisDataStreamRecord, + KinesisDataStreamRecordPayload, + ) + from .kinesis_firehose import ( + KinesisFirehoseModel, + KinesisFirehoseRecord, + KinesisFirehoseRecordMetadata, + ) + from .kinesis_firehose_sqs import KinesisFirehoseSqsModel, KinesisFirehoseSqsRecord + from .lambda_function_url import LambdaFunctionUrlModel + from .s3 import ( + S3EventNotificationEventBridgeDetailModel, + S3EventNotificationEventBridgeModel, + S3EventNotificationObjectModel, + S3Model, + S3RecordModel, + ) + from .s3_batch_operation import ( + S3BatchOperationJobModel, + S3BatchOperationModel, + S3BatchOperationTaskModel, + ) + from .s3_event_notification import ( + S3SqsEventNotificationModel, + S3SqsEventNotificationRecordModel, + ) + from .s3_object_event import ( + S3ObjectConfiguration, + S3ObjectContext, + S3ObjectLambdaEvent, + S3ObjectSessionAttributes, + S3ObjectSessionContext, + S3ObjectSessionIssuer, + S3ObjectUserIdentity, + S3ObjectUserRequest, + ) + from .ses import ( + SesMail, + SesMailCommonHeaders, + SesMailHeaders, + SesMessage, + SesModel, + SesReceipt, + SesReceiptAction, + SesReceiptVerdict, + SesRecordModel, + ) + from .sns import SnsModel, SnsNotificationModel, SnsRecordModel + from .sqs import SqsAttributesModel, SqsModel, SqsMsgAttributeModel, SqsRecordModel + from .transfer_family import TransferFamilyAuthorizer + from .vpc_lattice import VpcLatticeModel + from .vpc_latticev2 import VpcLatticeV2Model + + +_MODEL_MODULES = { + "AlbModel": "alb", + "AlbRequestContext": "alb", + "AlbRequestContextData": "alb", + "ApiGatewayAuthorizerRequest": "apigw", + "ApiGatewayAuthorizerToken": "apigw", + "APIGatewayEventAuthorizer": "apigw", + "APIGatewayEventIdentity": "apigw", + "APIGatewayEventRequestContext": "apigw", + "APIGatewayProxyEventModel": "apigw", + "APIGatewayWebSocketConnectEventModel": "apigw_websocket", + "APIGatewayWebSocketConnectEventRequestContext": "apigw_websocket", + "APIGatewayWebSocketDisconnectEventModel": "apigw_websocket", + "APIGatewayWebSocketDisconnectEventRequestContext": "apigw_websocket", + "APIGatewayWebSocketEventIdentity": "apigw_websocket", + "APIGatewayWebSocketEventRequestContextBase": "apigw_websocket", + "APIGatewayWebSocketMessageEventModel": "apigw_websocket", + "APIGatewayWebSocketMessageEventRequestContext": "apigw_websocket", + "ApiGatewayAuthorizerRequestV2": "apigwv2", + "APIGatewayProxyEventV2Model": "apigwv2", + "RequestContextV2": "apigwv2", + "RequestContextV2Authorizer": "apigwv2", + "RequestContextV2AuthorizerIam": "apigwv2", + "RequestContextV2AuthorizerIamCognito": "apigwv2", + "RequestContextV2AuthorizerJwt": "apigwv2", + "RequestContextV2Http": "apigwv2", + "AppSyncResolverEventModel": "appsync", + "AppSyncEventsModel": "appsync_events", + "BedrockAgentEventModel": "bedrock_agent", + "BedrockAgentFunctionEventModel": "bedrock_agent", + "BedrockAgentModel": "bedrock_agent", + "BedrockAgentPropertyModel": "bedrock_agent", + "BedrockAgentRequestBodyModel": "bedrock_agent", + "BedrockAgentRequestMediaModel": "bedrock_agent", + "CloudFormationCustomResourceBaseModel": "cloudformation_custom_resource", + "CloudFormationCustomResourceCreateModel": "cloudformation_custom_resource", + "CloudFormationCustomResourceDeleteModel": "cloudformation_custom_resource", + "CloudFormationCustomResourceUpdateModel": "cloudformation_custom_resource", + "CloudWatchLogsData": "cloudwatch", + "CloudWatchLogsDecode": "cloudwatch", + "CloudWatchLogsLogEvent": "cloudwatch", + "CloudWatchLogsModel": "cloudwatch", + "CognitoCreateAuthChallengeTriggerModel": "cognito", + "CognitoCustomEmailSenderTriggerModel": "cognito", + "CognitoCustomMessageTriggerModel": "cognito", + "CognitoCustomSMSSenderTriggerModel": "cognito", + "CognitoDefineAuthChallengeTriggerModel": "cognito", + "CognitoMigrateUserTriggerModel": "cognito", + "CognitoPostAuthenticationTriggerModel": "cognito", + "CognitoPostConfirmationTriggerModel": "cognito", + "CognitoPreAuthenticationTriggerModel": "cognito", + "CognitoPreSignupTriggerModel": "cognito", + "CognitoPreTokenGenerationTriggerModelV1": "cognito", + "CognitoPreTokenGenerationTriggerModelV2AndV3": "cognito", + "CognitoVerifyAuthChallengeTriggerModel": "cognito", + "DynamoDBStreamChangedRecordModel": "dynamodb", + "DynamoDBStreamLambdaOnFailureDestinationModel": "dynamodb", + "DynamoDBStreamModel": "dynamodb", + "DynamoDBStreamRecordModel": "dynamodb", + "EventBridgeModel": "event_bridge", + "KafkaBaseEventModel": "kafka", + "KafkaMskEventModel": "kafka", + "KafkaRecordModel": "kafka", + "KafkaSelfManagedEventModel": "kafka", + "KinesisDataStreamModel": "kinesis", + "KinesisDataStreamRecord": "kinesis", + "KinesisDataStreamRecordPayload": "kinesis", + "KinesisFirehoseModel": "kinesis_firehose", + "KinesisFirehoseRecord": "kinesis_firehose", + "KinesisFirehoseRecordMetadata": "kinesis_firehose", + "KinesisFirehoseSqsModel": "kinesis_firehose_sqs", + "KinesisFirehoseSqsRecord": "kinesis_firehose_sqs", + "LambdaFunctionUrlModel": "lambda_function_url", + "S3EventNotificationEventBridgeDetailModel": "s3", + "S3EventNotificationEventBridgeModel": "s3", + "S3EventNotificationObjectModel": "s3", + "S3Model": "s3", + "S3RecordModel": "s3", + "S3BatchOperationJobModel": "s3_batch_operation", + "S3BatchOperationModel": "s3_batch_operation", + "S3BatchOperationTaskModel": "s3_batch_operation", + "S3SqsEventNotificationModel": "s3_event_notification", + "S3SqsEventNotificationRecordModel": "s3_event_notification", + "S3ObjectConfiguration": "s3_object_event", + "S3ObjectContext": "s3_object_event", + "S3ObjectLambdaEvent": "s3_object_event", + "S3ObjectSessionAttributes": "s3_object_event", + "S3ObjectSessionContext": "s3_object_event", + "S3ObjectSessionIssuer": "s3_object_event", + "S3ObjectUserIdentity": "s3_object_event", + "S3ObjectUserRequest": "s3_object_event", + "SesMail": "ses", + "SesMailCommonHeaders": "ses", + "SesMailHeaders": "ses", + "SesMessage": "ses", + "SesModel": "ses", + "SesReceipt": "ses", + "SesReceiptAction": "ses", + "SesReceiptVerdict": "ses", + "SesRecordModel": "ses", + "SnsModel": "sns", + "SnsNotificationModel": "sns", + "SnsRecordModel": "sns", + "SqsAttributesModel": "sqs", + "SqsModel": "sqs", + "SqsMsgAttributeModel": "sqs", + "SqsRecordModel": "sqs", + "TransferFamilyAuthorizer": "transfer_family", + "VpcLatticeModel": "vpc_lattice", + "VpcLatticeV2Model": "vpc_latticev2", +} +_SUBMODULES = frozenset(_MODEL_MODULES.values()) + + +def __getattr__(name: str) -> object: + if name in _MODEL_MODULES: + module = importlib.import_module(f".{_MODEL_MODULES[name]}", __name__) + value = getattr(module, name) + elif name in _SUBMODULES: + value = importlib.import_module(f".{name}", __name__) + else: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + globals()[name] = value + return value + + +def __dir__() -> list[str]: + return sorted(set(globals()) | set(__all__) | _SUBMODULES) + __all__ = [ "APIGatewayProxyEventV2Model", diff --git a/tests/functional/parser/_model_import_probe.py b/tests/functional/parser/_model_import_probe.py new file mode 100644 index 00000000000..2e52094cb73 --- /dev/null +++ b/tests/functional/parser/_model_import_probe.py @@ -0,0 +1,86 @@ +import importlib +import inspect +import json +import pickle +import sys +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import get_args, get_type_hints + +PACKAGE = "aws_lambda_powertools.utilities.parser.models" +scenario = sys.argv[1] +models = importlib.import_module(PACKAGE) + + +def loaded_models(): + return {name.removeprefix(f"{PACKAGE}.") for name in sys.modules if name.startswith(f"{PACKAGE}.")} + + +if scenario == "lazy": + assert not loaded_models(), loaded_models() + assert set(models.__all__) | {"sqs", "alb"} <= set(dir(models)) + assert not loaded_models(), "dir() must not load models" + try: + _ = models.nonexistent_model + except AttributeError as exc: + assert str(exc) == f"module {PACKAGE!r} has no attribute 'nonexistent_model'" + else: + raise AssertionError("Unknown exports must raise AttributeError") + assert not loaded_models() +elif scenario in {"export", "submodule", "attribute"}: + name, module, *dependencies = sys.argv[2:] + if scenario == "export": + # Exercise Python's from-import handling as well as attribute lookup. + package = __import__(PACKAGE, fromlist=[name]) + model = getattr(package, name) + elif scenario == "submodule": + model = getattr(importlib.import_module(f"{PACKAGE}.{module}"), name) + else: + model = getattr(getattr(models, module), name) + assert loaded_models() == set(dependencies), loaded_models() + assert getattr(models, name) is model + assert vars(models)[name] is model, "The resolved export must be cached" + assert getattr(getattr(models, module), name) is model + assert loaded_models() == set(dependencies), loaded_models() +elif scenario == "exports": + for name in models.__all__: + model = getattr(models, name) + original = getattr(importlib.import_module(model.__module__), model.__name__) + assert model is original, name + assert pickle.loads(pickle.dumps(model)) is model, name + assert get_args(get_type_hints(models.SqsModel)["Records"]) == (models.SqsRecordModel,) +elif scenario == "star": + from aws_lambda_powertools.utilities.parser.models import * # noqa: E402,F403 + + for name in models.__all__: + assert globals()[name] is getattr(models, name), name +elif scenario == "introspection": + members = dict(inspect.getmembers(models)) + assert set(models.__all__) <= members.keys() + assert members["sqs"] is importlib.import_module(f"{PACKAGE}.sqs") + assert members["SqsModel"] is members["sqs"].SqsModel +elif scenario == "pickle": + from aws_lambda_powertools.utilities.parser.models import SqsModel + + event = json.loads((Path(__file__).parents[2] / "events" / "sqsEvent.json").read_text()) + parsed = SqsModel.model_validate(event) + restored = pickle.loads(pickle.dumps(parsed)) + assert type(restored) is SqsModel + assert restored.model_dump() == parsed.model_dump() + assert loaded_models() == {"sqs"} +elif scenario == "concurrent": + names = ["SqsModel", "S3Model", "S3SqsEventNotificationModel", "KinesisFirehoseSqsModel"] * 4 + with ThreadPoolExecutor(max_workers=8) as executor: + results = list(executor.map(lambda name: getattr(models, name), names)) + for name, model in zip(names, results, strict=True): + assert model is getattr(models, name), name + assert loaded_models() == { + "sqs", + "s3", + "event_bridge", + "s3_event_notification", + "kinesis_firehose_sqs", + "kinesis_firehose", + } +else: + raise ValueError(f"Unknown scenario: {scenario}") diff --git a/tests/functional/parser/test_model_imports.py b/tests/functional/parser/test_model_imports.py new file mode 100644 index 00000000000..c8a8a45006d --- /dev/null +++ b/tests/functional/parser/test_model_imports.py @@ -0,0 +1,51 @@ +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +MODEL_IMPORT_CASES = [ + ("SqsModel", "sqs", ["sqs"]), + ("S3Model", "s3", ["s3", "event_bridge"]), + ("AppSyncEventsModel", "appsync_events", ["appsync_events", "appsync"]), + ("LambdaFunctionUrlModel", "lambda_function_url", ["lambda_function_url", "apigwv2"]), + ( + "KinesisFirehoseSqsModel", + "kinesis_firehose_sqs", + ["kinesis_firehose_sqs", "kinesis_firehose", "sqs"], + ), + ( + "S3SqsEventNotificationModel", + "s3_event_notification", + ["s3_event_notification", "s3", "event_bridge", "sqs"], + ), +] + + +def run_probe(*args: str): + project_root = Path(__file__).parents[3] + probe = Path(__file__).with_name("_model_import_probe.py") + env = os.environ.copy() + env["PYTHONPATH"] = str(project_root) + result = subprocess.run( + [sys.executable, str(probe), *args], + cwd=project_root, + env=env, + capture_output=True, + text=True, + check=False, + timeout=30, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.mark.parametrize("access", ["export", "submodule", "attribute"]) +@pytest.mark.parametrize("name,module,dependencies", MODEL_IMPORT_CASES, ids=[case[0] for case in MODEL_IMPORT_CASES]) +def test_model_import_loads_only_its_dependencies(access, name, module, dependencies): + run_probe(access, name, module, *dependencies) + + +@pytest.mark.parametrize("scenario", ["lazy", "exports", "star", "introspection", "pickle", "concurrent"]) +def test_model_import_compatibility(scenario): + run_probe(scenario)