diff --git a/aws_lambda_powertools/utilities/parameters/base.py b/aws_lambda_powertools/utilities/parameters/base.py index 05067cb46b8..0343fd05343 100644 --- a/aws_lambda_powertools/utilities/parameters/base.py +++ b/aws_lambda_powertools/utilities/parameters/base.py @@ -206,6 +206,30 @@ def _get_multiple(self, path: str, **sdk_options) -> dict[str, str]: def clear_cache(self): self.store.clear() + def _invalidate_cache(self, name: str) -> None: + """Remove cached values for a single parameter or secret name, across all transforms + + Only entries holding a single value are removed. Grouped entries, created by + ``get_multiple``, hold the values of multiple names under a single key, so they + are kept intact to avoid evicting unrelated values. Use ``force_fetch=True`` + on the next read to refresh grouped results. + + Parameters + ---------- + name: str + Parameter or secret name whose cached values should be removed + """ + for key in list(self.store): + if key[0] != name: + continue + + # Grouped entries hold multiple values under one key; their third element is is_nested=True. + # Keys can also carry a fourth sdk_options element and can be two-element batch keys. + if len(key) >= 3 and key[2]: + continue + + del self.store[key] + def fetch_from_cache(self, key: tuple): return self.store[key].value if key in self.store else {} diff --git a/aws_lambda_powertools/utilities/parameters/secrets.py b/aws_lambda_powertools/utilities/parameters/secrets.py index eac469e2976..dc3f4b22eff 100644 --- a/aws_lambda_powertools/utilities/parameters/secrets.py +++ b/aws_lambda_powertools/utilities/parameters/secrets.py @@ -411,13 +411,18 @@ def set( try: logger.debug(f"Attempting to update secret {name}") - return self._update_secret(name=name, **sdk_options) + response = self._update_secret(name=name, **sdk_options) except self.client.exceptions.ResourceNotFoundException: logger.debug(f"Secret {name} doesn't exist, creating a new one") - return self._create_secret(name=name, **sdk_options) + response = self._create_secret(name=name, **sdk_options) except Exception as exc: raise SetSecretError(f"Error setting secret - {str(exc)}") from exc + # Remove any cached value for this secret so the next read returns what we just wrote + self._invalidate_cache(name) + + return response + @overload def get_secret( diff --git a/aws_lambda_powertools/utilities/parameters/ssm.py b/aws_lambda_powertools/utilities/parameters/ssm.py index bde6ef89e72..2d8add8d5d3 100644 --- a/aws_lambda_powertools/utilities/parameters/ssm.py +++ b/aws_lambda_powertools/utilities/parameters/ssm.py @@ -364,10 +364,15 @@ def set( opts["KeyId"] = kms_key_id try: - return self.client.put_parameter(**opts) + response = self.client.put_parameter(**opts) except Exception as exc: raise SetParameterError(f"Error setting parameter - {str(exc)}") from exc + # Remove any cached value for this parameter so the next read returns what we just wrote + self._invalidate_cache(name) + + return response + def _get(self, name: str, decrypt: bool = False, **sdk_options) -> str: """ Retrieve a parameter value from AWS Systems Manager Parameter Store diff --git a/tests/functional/parameters/_boto3/test_utilities_parameters.py b/tests/functional/parameters/_boto3/test_utilities_parameters.py index ce8c9c38aca..a5f0f1258ce 100644 --- a/tests/functional/parameters/_boto3/test_utilities_parameters.py +++ b/tests/functional/parameters/_boto3/test_utilities_parameters.py @@ -54,30 +54,52 @@ def mock_binary_value() -> str: return "ZXlKaGJHY2lPaUpJVXpJMU5pSXNJblI1Y0NJNklrcFhWQ0o5LmV5SnpkV0lpT2lJeE1qTTBOVFkzT0Rrd0lpd2libUZ0WlNJNklrcHZhRzRnUkc5bElpd2lhV0YwSWpveE5URTJNak01TURJeWZRLlNmbEt4d1JKU01lS0tGMlFUNGZ3cE1lSmYzNlBPazZ5SlZfYWRRc3N3NWMK" # noqa: E501 +class GetParametersStub(dict): + """ + GetParameters response where an omitted `InvalidParameters` key reads as an empty list. + + The SSM SDK model requires `InvalidParameters` to have at least one entry, so an empty + list fails `botocore.stub.Stubber` validation. AWS always returns the key, possibly + empty, and the provider reads it directly, so we omit it when empty and fall back to + an empty list on lookup. + """ + + def __missing__(self, key: str) -> list: + if key == "InvalidParameters": + return [] + raise KeyError(key) + + def build_get_parameters_stub( params: dict[str, Any], invalid_parameters: list[str] | None = None, ) -> dict[str, list]: invalid_parameters = invalid_parameters or [] version = random.randrange(1, 1000) - return { - "Parameters": [ - { - "Name": param, - "Type": "String", - "Value": value, - "Version": version, - # SSM only returns "Selector" for names requested with a version or label selector - "SourceResult": "string", - "LastModifiedDate": datetime(2015, 1, 1), - "ARN": f"arn:aws:ssm:us-east-2:111122223333:parameter/{param.lstrip('/')}", - "DataType": "string", - } - for param, value in params.items() - if param not in invalid_parameters - ], - "InvalidParameters": invalid_parameters, # official SDK stub fails validation here, need to raise an issue - } + stub = GetParametersStub( + { + "Parameters": [ + { + "Name": param, + "Type": "String", + "Value": value, + "Version": version, + # SSM only returns "Selector" for names requested with a version or label selector + "SourceResult": "string", + "LastModifiedDate": datetime(2015, 1, 1), + "ARN": f"arn:aws:ssm:us-east-2:111122223333:parameter/{param.lstrip('/')}", + "DataType": "string", + } + for param, value in params.items() + if param not in invalid_parameters + ], + }, + ) + + if invalid_parameters: + stub["InvalidParameters"] = invalid_parameters + + return stub def test_dynamodb_provider_get(mock_name, mock_value, config): @@ -707,6 +729,210 @@ def test_ssm_provider_set_parameter_raise_on_failure(mock_name, mock_value, mock stubber.deactivate() +def test_ssm_provider_set_parameter_invalidates_cached_value(mock_name, mock_value, mock_version, config): + """ + Test SSMProvider.set() removes a previously cached value so the next get returns the new one + """ + # GIVEN a provider that has cached a parameter value + provider = parameters.SSMProvider(boto_config=config) + new_value = f"{mock_value}-new" + + stubber = stub.Stubber(provider.client) + stubber.add_response( + "get_parameter", + { + "Parameter": { + "Name": mock_name, + "Type": "String", + "Value": mock_value, + "Version": mock_version, + "LastModifiedDate": datetime(2015, 1, 1), + "ARN": f"arn:aws:ssm:us-east-2:111122223333:parameter/{mock_name}", + }, + }, + {"Name": mock_name, "WithDecryption": False}, + ) + stubber.add_response( + "put_parameter", + {"Version": mock_version + 1, "Tier": "Standard"}, + { + "Name": mock_name, + "Value": new_value, + "Type": "String", + "Overwrite": True, + "Description": "", + "Tier": "Standard", + }, + ) + stubber.add_response( + "get_parameter", + { + "Parameter": { + "Name": mock_name, + "Type": "String", + "Value": new_value, + "Version": mock_version + 1, + "LastModifiedDate": datetime(2015, 1, 1), + "ARN": f"arn:aws:ssm:us-east-2:111122223333:parameter/{mock_name}", + }, + }, + {"Name": mock_name, "WithDecryption": False}, + ) + stubber.activate() + + try: + # WHEN reading the old value, writing a new one, and reading again + assert provider.get(mock_name) == mock_value + provider.set(name=mock_name, value=new_value, overwrite=True) + + # THEN the next read must hit SSM again and return the new value + assert provider.get(mock_name) == new_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + +def test_ssm_provider_set_parameter_invalidates_only_exact_name(mock_name, mock_value, config): + """ + Test SSMProvider.set() removes every cached transform of the exact name, preserving unrelated entries + """ + # GIVEN a provider with cached values for every transform of the name, keys + # carrying sdk_options, the two-element key used by get_parameters_by_name, + # grouped entries, and a similarly prefixed parameter name + provider = parameters.SSMProvider(boto_config=config) + similar_name = f"{mock_name}-similar" + + transform_keys = [ + provider._build_cache_key(name=mock_name, transform=transform) for transform in (None, "json", "binary", "auto") + ] + sdk_option_keys = [ + provider._build_cache_key(name=mock_name, transform=transform, sdk_options={"decrypt": False}) + for transform in (None, "json") + ] + batch_key = (mock_name, None) + single_value_keys = [*transform_keys, *sdk_option_keys, batch_key] + grouped_key = provider._build_cache_key(name=mock_name, transform=None, is_nested=True) + grouped_sdk_key = provider._build_cache_key( + name=mock_name, + transform=None, + is_nested=True, + sdk_options={"decrypt": False}, + ) + similar_key = provider._build_cache_key(name=similar_name) + + for key in single_value_keys: + provider.add_to_cache(key=key, value=mock_value, max_age=60) + provider.add_to_cache(key=grouped_key, value={similar_name: mock_value}, max_age=60) + provider.add_to_cache(key=grouped_sdk_key, value={similar_name: mock_value}, max_age=60) + provider.add_to_cache(key=similar_key, value=mock_value, max_age=60) + + stubber = stub.Stubber(provider.client) + stubber.add_response( + "put_parameter", + {"Version": 2, "Tier": "Standard"}, + { + "Name": mock_name, + "Value": mock_value, + "Type": "String", + "Overwrite": True, + "Description": "", + "Tier": "Standard", + }, + ) + stubber.activate() + + try: + # WHEN setting the parameter + provider.set(name=mock_name, value=mock_value, overwrite=True) + + # THEN every single-value entry for the exact name is removed + assert all(key not in provider.store for key in single_value_keys) + + # AND grouped entries (with or without sdk_options) and similarly prefixed names are kept + assert grouped_key in provider.store + assert grouped_sdk_key in provider.store + assert similar_key in provider.store + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + +def test_ssm_provider_set_parameter_failure_keeps_cache(mock_name, mock_value, config): + """ + Test SSMProvider.set() does not invalidate the cache when the write fails + """ + # GIVEN a provider with a cached value + provider = parameters.SSMProvider(boto_config=config) + cache_key = provider._build_cache_key(name=mock_name) + provider.add_to_cache(key=cache_key, value=mock_value, max_age=60) + + stubber = stub.Stubber(provider.client) + stubber.add_client_error( + "put_parameter", + service_error_code="InternalServerError", + service_message="Something went wrong", + http_status_code=500, + ) + stubber.activate() + + # WHEN the write fails + try: + with pytest.raises(parameters.exceptions.SetParameterError, match="Error setting parameter*"): + provider.set(name=mock_name, value=mock_value, overwrite=True) + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + # THEN the cached value is kept + assert provider.store[cache_key].value == mock_value + + +def test_ssm_provider_set_parameter_invalidates_get_parameters_by_name_cache(mock_name, mock_value, config): + """ + Test SSMProvider.set() removes the two-element cache key used by get_parameters_by_name + """ + # GIVEN a parameter cached through get_parameters_by_name + provider = parameters.SSMProvider(boto_config=config) + new_value = f"{mock_value}-new" + parameters_batch = {mock_name: {}} + + stubber = stub.Stubber(provider.client) + stubber.add_response( + "get_parameters", + build_get_parameters_stub(params={mock_name: mock_value}), + {"Names": [mock_name]}, + ) + stubber.add_response( + "put_parameter", + {"Version": 2, "Tier": "Standard"}, + { + "Name": mock_name, + "Value": new_value, + "Type": "String", + "Overwrite": True, + "Description": "", + "Tier": "Standard", + }, + ) + stubber.add_response( + "get_parameters", + build_get_parameters_stub(params={mock_name: new_value}), + {"Names": [mock_name]}, + ) + stubber.activate() + + try: + # WHEN reading the old value, writing a new one, and reading again + assert provider.get_parameters_by_name(parameters=parameters_batch)[mock_name] == mock_value + provider.set(name=mock_name, value=new_value, overwrite=True) + + # THEN the next batch read must hit SSM again and return the new value + assert provider.get_parameters_by_name(parameters=parameters_batch)[mock_name] == new_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + def test_set_secret(monkeypatch, mock_name, mock_value): """ Test set_secret() @@ -917,6 +1143,122 @@ def test_secret_provider_create_secret_raise_on_error(mocker, mock_name, mock_va stubber.deactivate() +def test_secret_provider_update_secret_invalidates_cached_value(mock_name, mock_value, config): + """ + Test SecretsProvider.set() removes a previously cached value when updating a secret + """ + # GIVEN a provider that has cached a secret value + provider = parameters.SecretsProvider(boto_config=config) + new_value = f"{mock_value}-new" + secret_arn = f"arn:aws:secretsmanager:us-east-1:132456789012:secret/{mock_name}" + + stubber = stub.Stubber(provider.client) + stubber.add_response( + "get_secret_value", + {"ARN": secret_arn, "Name": mock_name, "SecretString": mock_value}, + {"SecretId": mock_name}, + ) + stubber.add_response( + "put_secret_value", + {"ARN": secret_arn, "Name": mock_name}, + {"SecretId": mock_name, "SecretString": new_value}, + ) + stubber.add_response( + "get_secret_value", + {"ARN": secret_arn, "Name": mock_name, "SecretString": new_value}, + {"SecretId": mock_name}, + ) + stubber.activate() + + try: + # WHEN reading the old value, writing a new one, and reading again + assert provider.get(mock_name) == mock_value + provider.set(name=mock_name, value=new_value) + + # THEN the next read must hit Secrets Manager again and return the new value + assert provider.get(mock_name) == new_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + +def test_secret_provider_create_secret_invalidates_cached_value(mocker, mock_name, mock_value, config): + """ + Test SecretsProvider.set() removes a previously cached value when creating a secret + """ + # GIVEN a provider that has cached a secret value + provider = parameters.SecretsProvider(boto_config=config) + new_value = f"{mock_value}-new" + secret_arn = f"arn:aws:secretsmanager:us-east-1:132456789012:secret/{mock_name}" + + stubber = stub.Stubber(provider.client) + stubber.add_response( + "get_secret_value", + {"ARN": secret_arn, "Name": mock_name, "SecretString": mock_value}, + {"SecretId": mock_name}, + ) + stubber.add_response( + "create_secret", + {"ARN": secret_arn, "Name": mock_name}, + {"Name": mock_name, "SecretString": new_value}, + ) + stubber.add_response( + "get_secret_value", + {"ARN": secret_arn, "Name": mock_name, "SecretString": new_value}, + {"SecretId": mock_name}, + ) + stubber.activate() + + try: + # GIVEN a cached value for the secret + assert provider.get(mock_name) == mock_value + + # WHEN the secret doesn't exist yet and must be created + mock_update_secret = mocker.patch.object(provider, "_update_secret") + mock_update_secret.side_effect = provider.client.exceptions.ResourceNotFoundException( + {"Error": {"Code": "ResourceNotFoundException"}}, + "put_secret_value", + ) + + provider.set(name=mock_name, value=new_value) + + # THEN the next read must hit Secrets Manager again and return the new value + assert provider.get(mock_name) == new_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + +def test_secret_provider_set_failure_keeps_cache(mock_name, mock_value, config): + """ + Test SecretsProvider.set() does not invalidate the cache when the write fails + """ + # GIVEN a provider with a cached value + provider = parameters.SecretsProvider(boto_config=config) + cache_key = provider._build_cache_key(name=mock_name) + provider.add_to_cache(key=cache_key, value=mock_value, max_age=60) + + stubber = stub.Stubber(provider.client) + stubber.add_client_error( + "put_secret_value", + service_error_code="InternalServiceError", + service_message="Something went wrong", + http_status_code=500, + ) + stubber.activate() + + # WHEN the write fails + try: + with pytest.raises(parameters.exceptions.SetSecretError, match="Error setting secret*"): + provider.set(name=mock_name, value=mock_value) + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + # THEN the cached value is kept + assert provider.store[cache_key].value == mock_value + + def test_ssm_provider_get_with_custom_client(mock_name, mock_value, mock_version, config): """ Test SSMProvider.get() with a non-cached value