From de6745bd613db106b0e776fa1dba6050692a1507 Mon Sep 17 00:00:00 2001 From: Nicolas Tarzia Date: Wed, 23 Sep 2026 23:34:11 -0300 Subject: [PATCH 1/5] feat(data-masking): add Data Masking utility with Erase (Phase 1) Introduces the AWS.Lambda.Powertools.DataMasking package with the Erase operation for irreversibly masking sensitive fields in a payload. - DataMasking.Erase overloads: string and JsonNode (AOT-safe), object (annotated RequiresUnreferencedCode/RequiresDynamicCode), and object + JsonTypeInfo (AOT-safe via source generator) - MaskingOptions: default *****, fixed value, length-preserving, and regex + replacement rules - Field selection via simple dotted paths over System.Text.Json.Nodes (JMESPath expression support deferred to a later phase) - Unit tests (13) and an examples/DataMasking sample function Relates to #1257. Phase 1 (Erase) only; encryption/decryption via a pluggable provider (AWS Encryption SDK + KMS) planned for later phases. --- examples/DataMasking/README.md | 59 ++++++ examples/DataMasking/events/event.json | 10 + .../DataMasking/src/HelloWorld/Function.cs | 74 +++++++ .../src/HelloWorld/HelloWorld.csproj | 16 ++ .../HelloWorld/aws-lambda-tools-defaults.json | 12 ++ examples/DataMasking/template.yaml | 41 ++++ libraries/AWS.Lambda.Powertools.sln | 30 +++ .../AWS.Lambda.Powertools.DataMasking.csproj | 12 ++ .../DataMasking.cs | 120 +++++++++++ .../Internal/JsonNodeMasker.cs | 111 ++++++++++ .../InternalsVisibleTo.cs | 18 ++ .../MaskingOptions.cs | 98 +++++++++ .../README.md | 63 ++++++ ...Lambda.Powertools.DataMasking.Tests.csproj | 30 +++ .../EraseTests.cs | 195 ++++++++++++++++++ 15 files changed, 889 insertions(+) create mode 100644 examples/DataMasking/README.md create mode 100644 examples/DataMasking/events/event.json create mode 100644 examples/DataMasking/src/HelloWorld/Function.cs create mode 100644 examples/DataMasking/src/HelloWorld/HelloWorld.csproj create mode 100644 examples/DataMasking/src/HelloWorld/aws-lambda-tools-defaults.json create mode 100644 examples/DataMasking/template.yaml create mode 100644 libraries/src/AWS.Lambda.Powertools.DataMasking/AWS.Lambda.Powertools.DataMasking.csproj create mode 100644 libraries/src/AWS.Lambda.Powertools.DataMasking/DataMasking.cs create mode 100644 libraries/src/AWS.Lambda.Powertools.DataMasking/Internal/JsonNodeMasker.cs create mode 100644 libraries/src/AWS.Lambda.Powertools.DataMasking/InternalsVisibleTo.cs create mode 100644 libraries/src/AWS.Lambda.Powertools.DataMasking/MaskingOptions.cs create mode 100644 libraries/src/AWS.Lambda.Powertools.DataMasking/README.md create mode 100644 libraries/tests/AWS.Lambda.Powertools.DataMasking.Tests/AWS.Lambda.Powertools.DataMasking.Tests.csproj create mode 100644 libraries/tests/AWS.Lambda.Powertools.DataMasking.Tests/EraseTests.cs diff --git a/examples/DataMasking/README.md b/examples/DataMasking/README.md new file mode 100644 index 00000000..bbc2d2f9 --- /dev/null +++ b/examples/DataMasking/README.md @@ -0,0 +1,59 @@ +# Powertools for AWS Lambda (.NET) - Data Masking Example + +This example shows how to use the [Data Masking](https://github.com/aws-powertools/powertools-lambda-dotnet/issues/1257) +utility to irreversibly **erase** sensitive fields (PII) from a payload before it is logged or returned, +while keeping the non-sensitive fields readable. + +> This example covers **Phase 1 (`Erase`)** of the utility. Field-level and full-payload +> encryption/decryption via a pluggable provider (AWS Encryption SDK + KMS) are planned for later phases. + +## How it works + +The function receives a JSON order that contains PII (`ssn`, `creditCard`, `email`, `phone`, `street`) +and applies three masking styles: + +- **Full mask** (default `*****`) for `customer.ssn`, `payment.creditCard`, and `address.street` +- **Regex substitution** for `customer.email` (keeps the first character: `j****@example.com`) +- **Length-preserving mask** for `customer.phone` (`555-0134` -> `********`) + +The result keeps `orderId`, `item`, `payment.amount`, and `address.city` readable, so the payload +remains useful for debugging and monitoring. `LogEvent` is intentionally disabled so the raw PII is +never auto-logged. + +## Prerequisites + +- .NET 8.0 - [Install .NET 8.0](https://www.microsoft.com/net/download) +- [AWS SAM CLI](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/install-sam-cli.html) + +## Run locally + +Build and invoke the function with the sample event: + +```bash +sam build +sam local invoke HelloWorldFunction --event events/event.json +``` + +Expected (abbreviated) response body: + +```json +{ + "orderId": "ORD-1001", + "item": "Powertools T-Shirt", + "customer": { + "name": "John Doe", + "email": "j****@example.com", + "phone": "********", + "ssn": "*****" + }, + "address": { "street": "*****", "city": "Anytown" }, + "payment": { "creditCard": "*****", "amount": 42.5 } +} +``` + +## Deploy + +```bash +sam build +sam deploy --guided +``` diff --git a/examples/DataMasking/events/event.json b/examples/DataMasking/events/event.json new file mode 100644 index 00000000..867e8576 --- /dev/null +++ b/examples/DataMasking/events/event.json @@ -0,0 +1,10 @@ +{ + "resource": "/mask", + "path": "/mask", + "httpMethod": "POST", + "headers": { + "Content-Type": "application/json" + }, + "body": "{\"orderId\":\"ORD-1001\",\"item\":\"Powertools T-Shirt\",\"customer\":{\"name\":\"John Doe\",\"email\":\"john@example.com\",\"phone\":\"555-0134\",\"ssn\":\"123-45-6789\"},\"address\":{\"street\":\"123 Main St\",\"city\":\"Anytown\"},\"payment\":{\"creditCard\":\"4111111111111111\",\"amount\":42.5}}", + "isBase64Encoded": false +} diff --git a/examples/DataMasking/src/HelloWorld/Function.cs b/examples/DataMasking/src/HelloWorld/Function.cs new file mode 100644 index 00000000..7d50b6af --- /dev/null +++ b/examples/DataMasking/src/HelloWorld/Function.cs @@ -0,0 +1,74 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +using System.Collections.Generic; +using System.Text.RegularExpressions; +using Amazon.Lambda.APIGatewayEvents; +using Amazon.Lambda.Core; +using Amazon.Lambda.Serialization.SystemTextJson; +using AWS.Lambda.Powertools.DataMasking; +using AWS.Lambda.Powertools.Logging; + +// Assembly attribute to enable the Lambda function's JSON input to be converted into a .NET class. +[assembly: LambdaSerializer(typeof(DefaultLambdaJsonSerializer))] + +namespace HelloWorld; + +public class Function +{ + private readonly DataMasking _dataMasking = new(); + + /// + /// This example receives a JSON order payload that contains PII and demonstrates how to use the + /// Data Masking utility to irreversibly erase sensitive fields before the payload is logged and + /// returned. The non-sensitive fields (orderId, item, city) remain readable so the payload stays + /// useful for debugging and monitoring. + /// + [Logging(LogEvent = false)] // LogEvent is intentionally off so the raw PII is never auto-logged + public APIGatewayProxyResponse FunctionHandler(APIGatewayProxyRequest request, ILambdaContext context) + { + var body = request.Body ?? "{}"; + + // 1. Full-value masking (default *****) for whole fields + var masked = _dataMasking.Erase(body, new[] + { + "customer.ssn", + "payment.creditCard", + "address.street" + }); + + // 2. Custom rules: keep the first char of the email, and preserve the phone length + masked = _dataMasking.Erase(masked, new[] { "customer.email" }, new MaskingOptions + { + Pattern = new Regex("^(.).*@"), + Replacement = "$1****@" + }); + + masked = _dataMasking.Erase(masked, new[] { "customer.phone" }, new MaskingOptions + { + PreserveLength = true + }); + + // Safe to log now: sensitive fields are erased, structure is intact + Logger.LogInformation("Processing masked order: {Order}", masked); + + return new APIGatewayProxyResponse + { + Body = masked, + StatusCode = 200, + Headers = new Dictionary { { "Content-Type", "application/json" } } + }; + } +} diff --git a/examples/DataMasking/src/HelloWorld/HelloWorld.csproj b/examples/DataMasking/src/HelloWorld/HelloWorld.csproj new file mode 100644 index 00000000..efafdd55 --- /dev/null +++ b/examples/DataMasking/src/HelloWorld/HelloWorld.csproj @@ -0,0 +1,16 @@ + + + net8.0 + true + enable + + + + + + + + + + + diff --git a/examples/DataMasking/src/HelloWorld/aws-lambda-tools-defaults.json b/examples/DataMasking/src/HelloWorld/aws-lambda-tools-defaults.json new file mode 100644 index 00000000..a904dcf1 --- /dev/null +++ b/examples/DataMasking/src/HelloWorld/aws-lambda-tools-defaults.json @@ -0,0 +1,12 @@ +{ + "Information": [ + "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.", + "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.", + "dotnet lambda help", + "All the command line options for the Lambda command can be specified in this file." + ], + "profile": "", + "region": "", + "configuration": "Release", + "template": "template.yaml" +} diff --git a/examples/DataMasking/template.yaml b/examples/DataMasking/template.yaml new file mode 100644 index 00000000..c22cb2e1 --- /dev/null +++ b/examples/DataMasking/template.yaml @@ -0,0 +1,41 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 +AWSTemplateFormatVersion: "2010-09-09" +Transform: AWS::Serverless-2016-10-31 +Description: > + Example project for Powertools for AWS Lambda (.NET) Data Masking utility + +# More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst +Globals: + Function: + Timeout: 10 + Environment: + Variables: + POWERTOOLS_SERVICE_NAME: powertools-dotnet-data-masking-sample + POWERTOOLS_LOG_LEVEL: Information + +Resources: + HelloWorldFunction: + Type: AWS::Serverless::Function # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction + Properties: + Runtime: dotnet8 + CodeUri: ./src/HelloWorld/ + Handler: HelloWorld::HelloWorld.Function::FunctionHandler + MemorySize: 256 + Events: + MaskOrder: + Type: Api # More info about API Event Source: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#api + Properties: + Path: /mask + Method: post + +Outputs: + # ServerlessRestApi is an implicit API created out of Events key under Serverless::Function + # Find out more about other implicit resources you can reference within SAM + # https://github.com/awslabs/serverless-application-model/blob/master/docs/internals/generated_resources.rst#api + MaskOrderApi: + Description: "API Gateway endpoint URL for Prod stage for the Data Masking function" + Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/mask/" + HelloWorldFunction: + Description: "Data Masking Lambda Function ARN" + Value: !GetAtt HelloWorldFunction.Arn diff --git a/libraries/AWS.Lambda.Powertools.sln b/libraries/AWS.Lambda.Powertools.sln index d613a621..796c280c 100644 --- a/libraries/AWS.Lambda.Powertools.sln +++ b/libraries/AWS.Lambda.Powertools.sln @@ -131,6 +131,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AWS.Lambda.Powertools.Metad EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AWS.Lambda.Powertools.Metadata.Tests", "tests\AWS.Lambda.Powertools.Metadata.Tests\AWS.Lambda.Powertools.Metadata.Tests.csproj", "{B2ED85F6-FDBA-4AD7-8C13-1BF8BF464938}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AWS.Lambda.Powertools.DataMasking", "src\AWS.Lambda.Powertools.DataMasking\AWS.Lambda.Powertools.DataMasking.csproj", "{948193CD-A20C-4155-B60C-445FF4AE5D64}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AWS.Lambda.Powertools.DataMasking.Tests", "tests\AWS.Lambda.Powertools.DataMasking.Tests\AWS.Lambda.Powertools.DataMasking.Tests.csproj", "{F70E80CC-ACD5-4645-964D-653552621152}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -741,6 +745,30 @@ Global {B2ED85F6-FDBA-4AD7-8C13-1BF8BF464938}.Release|x64.Build.0 = Release|Any CPU {B2ED85F6-FDBA-4AD7-8C13-1BF8BF464938}.Release|x86.ActiveCfg = Release|Any CPU {B2ED85F6-FDBA-4AD7-8C13-1BF8BF464938}.Release|x86.Build.0 = Release|Any CPU + {948193CD-A20C-4155-B60C-445FF4AE5D64}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {948193CD-A20C-4155-B60C-445FF4AE5D64}.Debug|Any CPU.Build.0 = Debug|Any CPU + {948193CD-A20C-4155-B60C-445FF4AE5D64}.Debug|x64.ActiveCfg = Debug|Any CPU + {948193CD-A20C-4155-B60C-445FF4AE5D64}.Debug|x64.Build.0 = Debug|Any CPU + {948193CD-A20C-4155-B60C-445FF4AE5D64}.Debug|x86.ActiveCfg = Debug|Any CPU + {948193CD-A20C-4155-B60C-445FF4AE5D64}.Debug|x86.Build.0 = Debug|Any CPU + {948193CD-A20C-4155-B60C-445FF4AE5D64}.Release|Any CPU.ActiveCfg = Release|Any CPU + {948193CD-A20C-4155-B60C-445FF4AE5D64}.Release|Any CPU.Build.0 = Release|Any CPU + {948193CD-A20C-4155-B60C-445FF4AE5D64}.Release|x64.ActiveCfg = Release|Any CPU + {948193CD-A20C-4155-B60C-445FF4AE5D64}.Release|x64.Build.0 = Release|Any CPU + {948193CD-A20C-4155-B60C-445FF4AE5D64}.Release|x86.ActiveCfg = Release|Any CPU + {948193CD-A20C-4155-B60C-445FF4AE5D64}.Release|x86.Build.0 = Release|Any CPU + {F70E80CC-ACD5-4645-964D-653552621152}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F70E80CC-ACD5-4645-964D-653552621152}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F70E80CC-ACD5-4645-964D-653552621152}.Debug|x64.ActiveCfg = Debug|Any CPU + {F70E80CC-ACD5-4645-964D-653552621152}.Debug|x64.Build.0 = Debug|Any CPU + {F70E80CC-ACD5-4645-964D-653552621152}.Debug|x86.ActiveCfg = Debug|Any CPU + {F70E80CC-ACD5-4645-964D-653552621152}.Debug|x86.Build.0 = Debug|Any CPU + {F70E80CC-ACD5-4645-964D-653552621152}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F70E80CC-ACD5-4645-964D-653552621152}.Release|Any CPU.Build.0 = Release|Any CPU + {F70E80CC-ACD5-4645-964D-653552621152}.Release|x64.ActiveCfg = Release|Any CPU + {F70E80CC-ACD5-4645-964D-653552621152}.Release|x64.Build.0 = Release|Any CPU + {F70E80CC-ACD5-4645-964D-653552621152}.Release|x86.ActiveCfg = Release|Any CPU + {F70E80CC-ACD5-4645-964D-653552621152}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -805,5 +833,7 @@ Global {D2951A1A-D0EF-4CA4-AB4D-5ABAEFD164F5} = {1CFF5568-8486-475F-81F6-06105C437528} {6B978BB7-6C6E-481A-BE21-2E9E93B06AA0} = {73C9B1E5-3893-47E8-B373-17E5F5D7E6F5} {B2ED85F6-FDBA-4AD7-8C13-1BF8BF464938} = {1CFF5568-8486-475F-81F6-06105C437528} + {948193CD-A20C-4155-B60C-445FF4AE5D64} = {73C9B1E5-3893-47E8-B373-17E5F5D7E6F5} + {F70E80CC-ACD5-4645-964D-653552621152} = {1CFF5568-8486-475F-81F6-06105C437528} EndGlobalSection EndGlobal diff --git a/libraries/src/AWS.Lambda.Powertools.DataMasking/AWS.Lambda.Powertools.DataMasking.csproj b/libraries/src/AWS.Lambda.Powertools.DataMasking/AWS.Lambda.Powertools.DataMasking.csproj new file mode 100644 index 00000000..4289e7e0 --- /dev/null +++ b/libraries/src/AWS.Lambda.Powertools.DataMasking/AWS.Lambda.Powertools.DataMasking.csproj @@ -0,0 +1,12 @@ + + + + + AWS.Lambda.Powertools.DataMasking + Powertools for AWS Lambda (.NET) - Data Masking package. + AWS.Lambda.Powertools.DataMasking + AWS.Lambda.Powertools.DataMasking + enable + + + diff --git a/libraries/src/AWS.Lambda.Powertools.DataMasking/DataMasking.cs b/libraries/src/AWS.Lambda.Powertools.DataMasking/DataMasking.cs new file mode 100644 index 00000000..352183af --- /dev/null +++ b/libraries/src/AWS.Lambda.Powertools.DataMasking/DataMasking.cs @@ -0,0 +1,120 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization.Metadata; +using AWS.Lambda.Powertools.DataMasking.Internal; + +namespace AWS.Lambda.Powertools.DataMasking; + +/// +/// Provides irreversible masking (erasing) of sensitive fields within a payload. +/// +/// +/// This is Phase 1 of the Data Masking utility and covers Erase only. Field-level and +/// full-payload encryption/decryption via a pluggable provider are planned for later phases. +/// Field selection uses simple dotted paths (for example "address.street"). +/// +public sealed class DataMasking +{ + /// + /// Irreversibly masks the supplied fields within a JSON string. + /// This overload is fully trimming/AOT safe. + /// + /// The JSON payload as a string. + /// The dotted field paths to mask (for example "customer.ssn"). + /// Optional masking options. Defaults to a fixed ***** mask. + /// The masked payload serialized back to a JSON string. + /// or is . + public string Erase(string json, string[] fields, MaskingOptions? options = null) + { + if (json is null) throw new ArgumentNullException(nameof(json)); + if (fields is null) throw new ArgumentNullException(nameof(fields)); + + var root = JsonNode.Parse(json); + JsonNodeMasker.MaskFields(root, fields, options ?? MaskingOptions.Default); + return root?.ToJsonString() ?? "null"; + } + + /// + /// Irreversibly masks the supplied fields within a , mutating it in place. + /// This overload is fully trimming/AOT safe. + /// + /// The payload as a . Masked in place. + /// The dotted field paths to mask. + /// Optional masking options. Defaults to a fixed ***** mask. + /// The same instance, masked in place. + /// or is . + public JsonNode Erase(JsonNode node, string[] fields, MaskingOptions? options = null) + { + if (node is null) throw new ArgumentNullException(nameof(node)); + if (fields is null) throw new ArgumentNullException(nameof(fields)); + + JsonNodeMasker.MaskFields(node, fields, options ?? MaskingOptions.Default); + return node; + } + + /// + /// Irreversibly masks the supplied fields within an object, returning the masked payload as a JSON string. + /// + /// + /// This overload serializes using reflection-based + /// and is therefore not compatible with trimming/Native AOT. + /// For AOT scenarios use , + /// , or + /// with a source-generated context. + /// + /// The object to mask. + /// The dotted field paths to mask. + /// Optional masking options. Defaults to a fixed ***** mask. + /// The masked payload serialized to a JSON string. + /// or is . + [RequiresUnreferencedCode("JSON serialization of the supplied object might require types that cannot be statically analyzed. Use an overload that accepts a JsonTypeInfo for trimming/AOT scenarios.")] + [RequiresDynamicCode("JSON serialization of the supplied object might require runtime code generation. Use an overload that accepts a JsonTypeInfo for trimming/AOT scenarios.")] + public string Erase(object data, string[] fields, MaskingOptions? options = null) + { + if (data is null) throw new ArgumentNullException(nameof(data)); + if (fields is null) throw new ArgumentNullException(nameof(fields)); + + var root = JsonSerializer.SerializeToNode(data); + JsonNodeMasker.MaskFields(root, fields, options ?? MaskingOptions.Default); + return root?.ToJsonString() ?? "null"; + } + + /// + /// Irreversibly masks the supplied fields within an object using a source-generated + /// , returning the masked payload as a JSON string. + /// This overload is trimming/AOT safe when supplied with a source-generated type info. + /// + /// The object to mask. + /// The source-generated type metadata used to serialize . + /// The dotted field paths to mask. + /// Optional masking options. Defaults to a fixed ***** mask. + /// The masked payload serialized to a JSON string. + /// A required argument is . + public string Erase(object data, JsonTypeInfo jsonTypeInfo, string[] fields, MaskingOptions? options = null) + { + if (data is null) throw new ArgumentNullException(nameof(data)); + if (jsonTypeInfo is null) throw new ArgumentNullException(nameof(jsonTypeInfo)); + if (fields is null) throw new ArgumentNullException(nameof(fields)); + + var root = JsonSerializer.SerializeToNode(data, jsonTypeInfo); + JsonNodeMasker.MaskFields(root, fields, options ?? MaskingOptions.Default); + return root?.ToJsonString() ?? "null"; + } +} diff --git a/libraries/src/AWS.Lambda.Powertools.DataMasking/Internal/JsonNodeMasker.cs b/libraries/src/AWS.Lambda.Powertools.DataMasking/Internal/JsonNodeMasker.cs new file mode 100644 index 00000000..7984a90e --- /dev/null +++ b/libraries/src/AWS.Lambda.Powertools.DataMasking/Internal/JsonNodeMasker.cs @@ -0,0 +1,111 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Text.Json.Nodes; + +namespace AWS.Lambda.Powertools.DataMasking.Internal; + +/// +/// Applies masking to fields within a tree using simple dotted field paths +/// (for example "address.street"). +/// +/// +/// This intentionally operates on the object model rather than +/// reflecting over arbitrary CLR types, which keeps the operation trimming/AOT friendly. +/// Full JMESPath expression support (wildcards, filters) is planned for a later phase. +/// +internal static class JsonNodeMasker +{ + private const char PathSeparator = '.'; + + /// + /// Masks every supplied field path within in place. + /// + /// The root node of the payload being masked. + /// The dotted field paths to mask. + /// The masking options controlling the replacement. + internal static void MaskFields(JsonNode? root, IEnumerable fields, MaskingOptions options) + { + if (root is null) + { + return; + } + + foreach (var field in fields) + { + if (string.IsNullOrWhiteSpace(field)) + { + continue; + } + + MaskPath(root, field, options); + } + } + + private static void MaskPath(JsonNode root, string path, MaskingOptions options) + { + // Walk the dotted path segment-by-segment without allocating the intermediate string[] + // that string.Split would produce. JsonObject lookups require a string key, so each + // segment is still materialized, but the array (and its bounds/GC overhead) is avoided. + var current = root; + var start = 0; + + int separator; + while ((separator = path.IndexOf(PathSeparator, start)) >= 0) + { + var segment = path.Substring(start, separator - start); + + // If any intermediate segment is missing or is not an object, the path does not + // exist in this payload and is skipped. + if (current is not JsonObject obj || !obj.TryGetPropertyValue(segment, out var next) || next is null) + { + return; + } + + current = next; + start = separator + 1; + } + + if (current is JsonObject parent) + { + var leaf = start == 0 ? path : path.Substring(start); + if (parent.ContainsKey(leaf)) + { + parent[leaf] = MaskValue(parent[leaf], options); + } + } + } + + private static JsonNode? MaskValue(JsonNode? value, MaskingOptions options) + { + // A null JSON value stays null; there is nothing sensitive to mask. + if (value is null) + { + return null; + } + + // The fixed-mask case (the default) ignores the original value, so avoid serializing + // the (potentially large) subtree to a string just to discard it. + if (!options.RequiresRawValue) + { + return JsonValue.Create(options.Apply(string.Empty)); + } + + var raw = value is JsonValue jsonValue ? jsonValue.ToString() : value.ToJsonString(); + return JsonValue.Create(options.Apply(raw)); + } +} diff --git a/libraries/src/AWS.Lambda.Powertools.DataMasking/InternalsVisibleTo.cs b/libraries/src/AWS.Lambda.Powertools.DataMasking/InternalsVisibleTo.cs new file mode 100644 index 00000000..d5e2173f --- /dev/null +++ b/libraries/src/AWS.Lambda.Powertools.DataMasking/InternalsVisibleTo.cs @@ -0,0 +1,18 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("AWS.Lambda.Powertools.DataMasking.Tests")] diff --git a/libraries/src/AWS.Lambda.Powertools.DataMasking/MaskingOptions.cs b/libraries/src/AWS.Lambda.Powertools.DataMasking/MaskingOptions.cs new file mode 100644 index 00000000..4d7bd9a8 --- /dev/null +++ b/libraries/src/AWS.Lambda.Powertools.DataMasking/MaskingOptions.cs @@ -0,0 +1,98 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +using System.Text.RegularExpressions; + +namespace AWS.Lambda.Powertools.DataMasking; + +/// +/// Controls how values are masked by . +/// +/// +/// By default a fixed ***** string replaces the entire value. The options below allow +/// customizing the masking behavior while keeping the operation irreversible. +/// +public sealed class MaskingOptions +{ + /// + /// The default masking value applied when no custom rule is configured. + /// + public const string DefaultMask = "*****"; + + /// + /// The fixed string used to replace a value. + /// Ignored when is true or when is set. + /// Defaults to . + /// + public string MaskValue { get; set; } = DefaultMask; + + /// + /// When true, the value is replaced by a mask that preserves the original character length + /// (for example "secret" becomes "******"), using . + /// Ignored when is set. + /// + public bool PreserveLength { get; set; } + + /// + /// The character used when is enabled. Defaults to '*'. + /// + public char MaskCharacter { get; set; } = '*'; + + /// + /// An optional regular expression applied to the string representation of the value. + /// Matches are replaced with . When set, this takes precedence + /// over and . + /// + public Regex? Pattern { get; set; } + + /// + /// The replacement applied to matches of . Supports regex substitution + /// syntax (for example "$1****"). Defaults to . + /// + public string Replacement { get; set; } = DefaultMask; + + /// + /// The default options: replace the whole value with . + /// + public static MaskingOptions Default { get; } = new(); + + /// + /// Whether reads the original value. This is only the case when a + /// is set or is enabled; for the default + /// fixed- behavior the original value is ignored and callers can skip + /// materializing it. + /// + internal bool RequiresRawValue => Pattern is not null || PreserveLength; + + /// + /// Produces the masked representation of the supplied raw string value according to these options. + /// + /// The original value's string representation. + /// The masked string. + internal string Apply(string value) + { + if (Pattern is not null) + { + return Pattern.Replace(value, Replacement); + } + + if (PreserveLength) + { + return new string(MaskCharacter, value.Length); + } + + return MaskValue; + } +} diff --git a/libraries/src/AWS.Lambda.Powertools.DataMasking/README.md b/libraries/src/AWS.Lambda.Powertools.DataMasking/README.md new file mode 100644 index 00000000..66ae92f9 --- /dev/null +++ b/libraries/src/AWS.Lambda.Powertools.DataMasking/README.md @@ -0,0 +1,63 @@ +# Powertools for AWS Lambda (.NET) - Data Masking + +The Data Masking utility protects sensitive data (PII, credentials, financial records) before it is +logged, stored, or forwarded to another service, helping to meet requirements such as GDPR, HIPAA, +and PCI-DSS. + +> **Status:** this package currently ships **Phase 1 – `Erase`** (irreversible masking). Field-level +> and full-payload encryption/decryption through a pluggable provider (AWS Encryption SDK + KMS) are +> planned for later phases. See [issue #1257](https://github.com/aws-powertools/powertools-lambda-dotnet/issues/1257). + +## Key features + +- Irreversibly **erase** sensitive fields while keeping the surrounding payload structure intact +- Select fields with simple dotted paths (for example `address.street`) +- Customizable masking: fixed string, length-preserving mask, or regex + replacement +- Trimming/Native AOT friendly when operating over JSON strings or `JsonNode` + +## Erase + +```csharp +using AWS.Lambda.Powertools.DataMasking; + +var masker = new DataMasking(); + +string json = """ +{ + "email": "john@example.com", + "address": { "street": "123 Main St", "city": "Anytown" }, + "customer": { "ssn": "123-45-6789" } +} +"""; + +// Mask specific fields; the rest of the payload stays readable +string masked = masker.Erase(json, fields: new[] { "email", "address.street", "customer.ssn" }); +// { +// "email": "*****", +// "address": { "street": "*****", "city": "Anytown" }, +// "customer": { "ssn": "*****" } +// } +``` + +### Custom masking + +```csharp +// Length-preserving mask: "secret" -> "******" +masker.Erase(json, new[] { "password" }, new MaskingOptions { PreserveLength = true }); + +// Fixed replacement string +masker.Erase(json, new[] { "token" }, new MaskingOptions { MaskValue = "[REDACTED]" }); + +// Regex + replacement (keeps the first character of an email local part) +masker.Erase(json, new[] { "email" }, new MaskingOptions +{ + Pattern = new System.Text.RegularExpressions.Regex(@"^(.).*@"), + Replacement = "$1****@" +}); +``` + +### AOT / trimming + +The `Erase(string, ...)` and `Erase(JsonNode, ...)` overloads are fully trimming/AOT safe. The +`Erase(object, ...)` overload uses reflection-based serialization and is annotated accordingly; for +AOT scenarios use the overload that accepts a source-generated `JsonTypeInfo`. diff --git a/libraries/tests/AWS.Lambda.Powertools.DataMasking.Tests/AWS.Lambda.Powertools.DataMasking.Tests.csproj b/libraries/tests/AWS.Lambda.Powertools.DataMasking.Tests/AWS.Lambda.Powertools.DataMasking.Tests.csproj new file mode 100644 index 00000000..8c5bffd3 --- /dev/null +++ b/libraries/tests/AWS.Lambda.Powertools.DataMasking.Tests/AWS.Lambda.Powertools.DataMasking.Tests.csproj @@ -0,0 +1,30 @@ + + + + default + AWS.Lambda.Powertools.DataMasking.Tests + AWS.Lambda.Powertools.DataMasking.Tests + enable + + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + diff --git a/libraries/tests/AWS.Lambda.Powertools.DataMasking.Tests/EraseTests.cs b/libraries/tests/AWS.Lambda.Powertools.DataMasking.Tests/EraseTests.cs new file mode 100644 index 00000000..a61a4dcc --- /dev/null +++ b/libraries/tests/AWS.Lambda.Powertools.DataMasking.Tests/EraseTests.cs @@ -0,0 +1,195 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +using System; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using FluentAssertions; +using Xunit; + +namespace AWS.Lambda.Powertools.DataMasking.Tests; + +public class EraseTests +{ + private const string SampleJson = """ + { + "email": "john@example.com", + "address": { "street": "123 Main St", "city": "Anytown" }, + "customer": { "ssn": "123-45-6789" } + } + """; + + [Fact] + public void Erase_TopLevelField_ReplacesWithDefaultMask() + { + var masker = new DataMasking(); + + var result = masker.Erase(SampleJson, new[] { "email" }); + + var node = JsonNode.Parse(result)!; + node["email"]!.GetValue().Should().Be("*****"); + } + + [Fact] + public void Erase_NestedField_MasksOnlyThatField_AndKeepsSiblings() + { + var masker = new DataMasking(); + + var result = masker.Erase(SampleJson, new[] { "address.street" }); + + var node = JsonNode.Parse(result)!; + node["address"]!["street"]!.GetValue().Should().Be("*****"); + // Sibling remains readable + node["address"]!["city"]!.GetValue().Should().Be("Anytown"); + } + + [Fact] + public void Erase_MultipleFields_MasksAllOfThem() + { + var masker = new DataMasking(); + + var result = masker.Erase(SampleJson, new[] { "email", "address.street", "customer.ssn" }); + + var node = JsonNode.Parse(result)!; + node["email"]!.GetValue().Should().Be("*****"); + node["address"]!["street"]!.GetValue().Should().Be("*****"); + node["customer"]!["ssn"]!.GetValue().Should().Be("*****"); + node["address"]!["city"]!.GetValue().Should().Be("Anytown"); + } + + [Fact] + public void Erase_MissingField_IsSkipped_WithoutError() + { + var masker = new DataMasking(); + + var result = masker.Erase(SampleJson, new[] { "does.not.exist", "email" }); + + var node = JsonNode.Parse(result)!; + // Existing field still masked, missing path silently ignored + node["email"]!.GetValue().Should().Be("*****"); + node["address"]!["city"]!.GetValue().Should().Be("Anytown"); + } + + [Fact] + public void Erase_PathThroughNonObject_IsSkipped() + { + var masker = new DataMasking(); + + // "email" is a string, so "email.something" cannot be traversed + var result = masker.Erase(SampleJson, new[] { "email.something" }); + + var node = JsonNode.Parse(result)!; + node["email"]!.GetValue().Should().Be("john@example.com"); + } + + [Fact] + public void Erase_PreserveLength_UsesMaskCharacterMatchingOriginalLength() + { + var masker = new DataMasking(); + var options = new MaskingOptions { PreserveLength = true }; + + var result = masker.Erase(SampleJson, new[] { "customer.ssn" }, options); + + var node = JsonNode.Parse(result)!; + // "123-45-6789" has 11 characters + node["customer"]!["ssn"]!.GetValue().Should().Be(new string('*', "123-45-6789".Length)); + } + + [Fact] + public void Erase_CustomMaskValue_IsUsed() + { + var masker = new DataMasking(); + var options = new MaskingOptions { MaskValue = "[REDACTED]" }; + + var result = masker.Erase(SampleJson, new[] { "email" }, options); + + var node = JsonNode.Parse(result)!; + node["email"]!.GetValue().Should().Be("[REDACTED]"); + } + + [Fact] + public void Erase_WithRegexPattern_AppliesSubstitution() + { + var masker = new DataMasking(); + var options = new MaskingOptions + { + Pattern = new Regex(@"^(.).*@"), + Replacement = "$1****@" + }; + + var result = masker.Erase(SampleJson, new[] { "email" }, options); + + var node = JsonNode.Parse(result)!; + node["email"]!.GetValue().Should().Be("j****@example.com"); + } + + [Fact] + public void Erase_JsonNodeOverload_MasksInPlace_AndReturnsSameInstance() + { + var masker = new DataMasking(); + var node = JsonNode.Parse(SampleJson)!; + + var returned = masker.Erase(node, new[] { "email" }); + + returned.Should().BeSameAs(node); + node["email"]!.GetValue().Should().Be("*****"); + } + + [Fact] + public void Erase_ObjectOverload_SerializesAndMasks() + { + var masker = new DataMasking(); + var data = new { email = "john@example.com", name = "John" }; + +#pragma warning disable IL2026, IL3050 // Reflection-based overload exercised intentionally in tests + var result = masker.Erase(data, new[] { "email" }); +#pragma warning restore IL2026, IL3050 + + var node = JsonNode.Parse(result)!; + node["email"]!.GetValue().Should().Be("*****"); + node["name"]!.GetValue().Should().Be("John"); + } + + [Fact] + public void Erase_NullJson_Throws() + { + var masker = new DataMasking(); + + var act = () => masker.Erase((string)null!, new[] { "email" }); + + act.Should().Throw(); + } + + [Fact] + public void Erase_NullFields_Throws() + { + var masker = new DataMasking(); + + var act = () => masker.Erase(SampleJson, null!); + + act.Should().Throw(); + } + + [Fact] + public void Erase_EmptyOrWhitespaceFieldPaths_AreIgnored() + { + var masker = new DataMasking(); + + var result = masker.Erase(SampleJson, new[] { "", " ", "email" }); + + var node = JsonNode.Parse(result)!; + node["email"]!.GetValue().Should().Be("*****"); + } +} From 44eb8d6bac0b81eff5407fe31990c42e9a4bd1e7 Mon Sep 17 00:00:00 2001 From: Nicolas Tarzia Date: Wed, 23 Sep 2026 23:59:37 -0300 Subject: [PATCH 2/5] feat(data-masking): add Encrypt/Decrypt with KMS provider (Phase 2) Adds reversible encryption to the Data Masking utility via a pluggable provider, with a default AWS KMS envelope-encryption implementation. - IDataMaskingProvider abstraction (EncryptAsync/DecryptAsync with encryption context and cancellation token) - AwsEncryptionSdkProvider: default provider using the AWS Encryption SDK for .NET; single-key (CreateAwsKmsKeyring) or multi-key (multi-keyring); supports a custom KMS client. Annotated RequiresUnreferencedCode/ RequiresDynamicCode since the SDK's Dafny runtime is not AOT-safe. - DataMasking: optional ctor(IDataMaskingProvider) plus EncryptAsync/ DecryptAsync for whole-payload and field-level encryption over JsonNode - JsonNodeMasker.TransformFieldsAsync for async field transforms - Central Package Management: AWS.Cryptography.EncryptionSDK 5.0.0 and AWSSDK.KeyManagementService - Tests: FakeDataMaskingProvider (reversible, context-binding) and 8 Encrypt/Decrypt tests (round-trip, field-level, context validation, missing-provider guards). 21 tests total pass on net10.0. Relates to #1257. Phase 2 (Encrypt/Decrypt) building on Phase 1 (Erase). The AwsEncryptionSdkProvider is not covered by unit tests as it requires live AWS KMS credentials. --- .../AWS.Lambda.Powertools.DataMasking.csproj | 6 + .../DataMasking.cs | 128 +++++++++++++- .../Internal/JsonNodeMasker.cs | 57 +++++++ .../Providers/AwsEncryptionSdkProvider.cs | 157 ++++++++++++++++++ .../Providers/IDataMaskingProvider.cs | 59 +++++++ .../README.md | 42 ++++- libraries/src/Directory.Packages.props | 2 + .../EncryptDecryptTests.cs | 138 +++++++++++++++ .../FakeDataMaskingProvider.cs | 84 ++++++++++ 9 files changed, 665 insertions(+), 8 deletions(-) create mode 100644 libraries/src/AWS.Lambda.Powertools.DataMasking/Providers/AwsEncryptionSdkProvider.cs create mode 100644 libraries/src/AWS.Lambda.Powertools.DataMasking/Providers/IDataMaskingProvider.cs create mode 100644 libraries/tests/AWS.Lambda.Powertools.DataMasking.Tests/EncryptDecryptTests.cs create mode 100644 libraries/tests/AWS.Lambda.Powertools.DataMasking.Tests/FakeDataMaskingProvider.cs diff --git a/libraries/src/AWS.Lambda.Powertools.DataMasking/AWS.Lambda.Powertools.DataMasking.csproj b/libraries/src/AWS.Lambda.Powertools.DataMasking/AWS.Lambda.Powertools.DataMasking.csproj index 4289e7e0..d2065ae8 100644 --- a/libraries/src/AWS.Lambda.Powertools.DataMasking/AWS.Lambda.Powertools.DataMasking.csproj +++ b/libraries/src/AWS.Lambda.Powertools.DataMasking/AWS.Lambda.Powertools.DataMasking.csproj @@ -9,4 +9,10 @@ enable + + + + + + diff --git a/libraries/src/AWS.Lambda.Powertools.DataMasking/DataMasking.cs b/libraries/src/AWS.Lambda.Powertools.DataMasking/DataMasking.cs index 352183af..6c0e0edd 100644 --- a/libraries/src/AWS.Lambda.Powertools.DataMasking/DataMasking.cs +++ b/libraries/src/AWS.Lambda.Powertools.DataMasking/DataMasking.cs @@ -14,24 +14,49 @@ */ using System; +using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Text.Json; using System.Text.Json.Nodes; using System.Text.Json.Serialization.Metadata; +using System.Threading; +using System.Threading.Tasks; using AWS.Lambda.Powertools.DataMasking.Internal; +using AWS.Lambda.Powertools.DataMasking.Providers; namespace AWS.Lambda.Powertools.DataMasking; /// -/// Provides irreversible masking (erasing) of sensitive fields within a payload. +/// Provides masking of sensitive fields within a payload: irreversible Erase, and reversible +/// Encrypt/Decrypt when constructed with an . /// /// -/// This is Phase 1 of the Data Masking utility and covers Erase only. Field-level and -/// full-payload encryption/decryption via a pluggable provider are planned for later phases. -/// Field selection uses simple dotted paths (for example "address.street"). +/// Field selection uses simple dotted paths (for example "address.street"). Full JMESPath +/// expression support is planned for a later phase. /// public sealed class DataMasking { + private readonly IDataMaskingProvider? _provider; + + /// + /// Creates an instance that supports the Erase operations only. + /// Use to enable Encrypt/Decrypt. + /// + public DataMasking() + { + } + + /// + /// Creates an instance backed by an encryption provider, enabling Encrypt/Decrypt + /// in addition to Erase. + /// + /// The encryption backend (for example AwsEncryptionSdkProvider). + /// is . + public DataMasking(IDataMaskingProvider provider) + { + _provider = provider ?? throw new ArgumentNullException(nameof(provider)); + } + /// /// Irreversibly masks the supplied fields within a JSON string. /// This overload is fully trimming/AOT safe. @@ -117,4 +142,99 @@ public string Erase(object data, JsonTypeInfo jsonTypeInfo, string[] fields, Mas JsonNodeMasker.MaskFields(root, fields, options ?? MaskingOptions.Default); return root?.ToJsonString() ?? "null"; } + + /// + /// Encrypts the entire JSON payload using the configured . + /// + /// The JSON payload as a string. + /// Optional encryption context bound to the ciphertext. + /// A token to cancel the operation. + /// The encrypted payload. + /// is . + /// No provider was configured. + public Task EncryptAsync( + string json, + IDictionary? encryptionContext = null, + CancellationToken cancellationToken = default) + { + if (json is null) throw new ArgumentNullException(nameof(json)); + return RequireProvider().EncryptAsync(json, encryptionContext, cancellationToken); + } + + /// + /// Encrypts only the supplied fields within a JSON payload, leaving the rest readable. + /// + /// The JSON payload as a string. + /// The dotted field paths to encrypt. + /// Optional encryption context bound to each field's ciphertext. + /// A token to cancel the operation. + /// The payload with the selected fields encrypted, serialized to a JSON string. + /// or is . + /// No provider was configured. + public async Task EncryptAsync( + string json, + string[] fields, + IDictionary? encryptionContext = null, + CancellationToken cancellationToken = default) + { + if (json is null) throw new ArgumentNullException(nameof(json)); + if (fields is null) throw new ArgumentNullException(nameof(fields)); + + var provider = RequireProvider(); + var root = JsonNode.Parse(json); + await JsonNodeMasker.TransformFieldsAsync(root, fields, + value => provider.EncryptAsync(value, encryptionContext, cancellationToken)); + return root?.ToJsonString() ?? "null"; + } + + /// + /// Decrypts an entire payload previously produced by . + /// + /// The encrypted payload. + /// Optional encryption context to validate against the value bound at encrypt time. + /// A token to cancel the operation. + /// The decrypted payload. + /// is . + /// No provider was configured. + public Task DecryptAsync( + string value, + IDictionary? encryptionContext = null, + CancellationToken cancellationToken = default) + { + if (value is null) throw new ArgumentNullException(nameof(value)); + return RequireProvider().DecryptAsync(value, encryptionContext, cancellationToken); + } + + /// + /// Decrypts only the supplied fields within a JSON payload previously produced by + /// . + /// + /// The JSON payload as a string. + /// The dotted field paths to decrypt. + /// Optional encryption context to validate. + /// A token to cancel the operation. + /// The payload with the selected fields decrypted, serialized to a JSON string. + /// or is . + /// No provider was configured. + public async Task DecryptAsync( + string json, + string[] fields, + IDictionary? encryptionContext = null, + CancellationToken cancellationToken = default) + { + if (json is null) throw new ArgumentNullException(nameof(json)); + if (fields is null) throw new ArgumentNullException(nameof(fields)); + + var provider = RequireProvider(); + var root = JsonNode.Parse(json); + await JsonNodeMasker.TransformFieldsAsync(root, fields, + value => provider.DecryptAsync(value, encryptionContext, cancellationToken)); + return root?.ToJsonString() ?? "null"; + } + + private IDataMaskingProvider RequireProvider() + { + return _provider ?? throw new InvalidOperationException( + "This operation requires an encryption provider. Construct DataMasking with an IDataMaskingProvider (for example new DataMasking(new AwsEncryptionSdkProvider(keyArn)))."); + } } diff --git a/libraries/src/AWS.Lambda.Powertools.DataMasking/Internal/JsonNodeMasker.cs b/libraries/src/AWS.Lambda.Powertools.DataMasking/Internal/JsonNodeMasker.cs index 7984a90e..04e48e7e 100644 --- a/libraries/src/AWS.Lambda.Powertools.DataMasking/Internal/JsonNodeMasker.cs +++ b/libraries/src/AWS.Lambda.Powertools.DataMasking/Internal/JsonNodeMasker.cs @@ -16,6 +16,7 @@ using System; using System.Collections.Generic; using System.Text.Json.Nodes; +using System.Threading.Tasks; namespace AWS.Lambda.Powertools.DataMasking.Internal; @@ -108,4 +109,60 @@ private static void MaskPath(JsonNode root, string path, MaskingOptions options) var raw = value is JsonValue jsonValue ? jsonValue.ToString() : value.ToJsonString(); return JsonValue.Create(options.Apply(raw)); } + + /// + /// Applies an asynchronous string transform (for example encrypt or decrypt) to each supplied + /// field path within , in place. Missing paths are skipped. + /// + internal static async Task TransformFieldsAsync( + JsonNode? root, + IEnumerable fields, + Func> transform) + { + if (root is null) + { + return; + } + + foreach (var field in fields) + { + if (string.IsNullOrWhiteSpace(field)) + { + continue; + } + + await TransformPathAsync(root, field, transform); + } + } + + private static async Task TransformPathAsync(JsonNode root, string path, Func> transform) + { + var current = root; + var start = 0; + + int separator; + while ((separator = path.IndexOf(PathSeparator, start)) >= 0) + { + var segment = path.Substring(start, separator - start); + if (current is not JsonObject obj || !obj.TryGetPropertyValue(segment, out var next) || next is null) + { + return; + } + + current = next; + start = separator + 1; + } + + if (current is JsonObject parent) + { + var leaf = start == 0 ? path : path.Substring(start); + if (parent.ContainsKey(leaf) && parent[leaf] is not null) + { + var value = parent[leaf]!; + var raw = value is JsonValue jsonValue ? jsonValue.ToString() : value.ToJsonString(); + var transformed = await transform(raw); + parent[leaf] = JsonValue.Create(transformed); + } + } + } } diff --git a/libraries/src/AWS.Lambda.Powertools.DataMasking/Providers/AwsEncryptionSdkProvider.cs b/libraries/src/AWS.Lambda.Powertools.DataMasking/Providers/AwsEncryptionSdkProvider.cs new file mode 100644 index 00000000..2a13edb8 --- /dev/null +++ b/libraries/src/AWS.Lambda.Powertools.DataMasking/Providers/AwsEncryptionSdkProvider.cs @@ -0,0 +1,157 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Amazon.KeyManagementService; +using AWS.Cryptography.EncryptionSDK; +using AWS.Cryptography.MaterialProviders; + +namespace AWS.Lambda.Powertools.DataMasking.Providers; + +/// +/// Default that performs AWS KMS envelope encryption using the +/// AWS Encryption SDK for .NET. +/// +/// +/// This provider relies on the AWS Encryption SDK for .NET (and its Dafny-based runtime), which uses +/// runtime reflection. It is therefore not compatible with trimming or Native AOT. The type is +/// annotated accordingly. +/// +public sealed class AwsEncryptionSdkProvider : IDataMaskingProvider +{ + private readonly ESDK _encryptionSdk; + private readonly IKeyring _keyring; + + /// + /// Creates a provider that protects data with the supplied AWS KMS key(s). + /// + /// + /// One or more KMS key identifiers. For encryption you can use a key ID, key ARN, alias name, or + /// alias ARN. For decryption in strict mode a key ARN is required, so prefer key ARNs if the same + /// keyring is reused for decryption. + /// + /// is . + /// is empty. + [RequiresUnreferencedCode("The AWS Encryption SDK for .NET uses runtime reflection and is not compatible with trimming.")] + [RequiresDynamicCode("The AWS Encryption SDK for .NET uses runtime code generation and is not compatible with Native AOT.")] + public AwsEncryptionSdkProvider(params string[] keyIds) + : this(new AmazonKeyManagementServiceClient(), keyIds) + { + } + + /// + /// Creates a provider with a caller-supplied KMS client (useful for a specific Region or for testing). + /// + /// The KMS client to use. + /// One or more KMS key identifiers (see the other constructor for guidance). + /// A required argument is . + /// is empty. + [RequiresUnreferencedCode("The AWS Encryption SDK for .NET uses runtime reflection and is not compatible with trimming.")] + [RequiresDynamicCode("The AWS Encryption SDK for .NET uses runtime code generation and is not compatible with Native AOT.")] + public AwsEncryptionSdkProvider(IAmazonKeyManagementService kmsClient, params string[] keyIds) + { + if (kmsClient is null) throw new ArgumentNullException(nameof(kmsClient)); + if (keyIds is null) throw new ArgumentNullException(nameof(keyIds)); + if (keyIds.Length == 0) throw new ArgumentException("At least one KMS key identifier is required.", nameof(keyIds)); + + _encryptionSdk = new ESDK(new AwsEncryptionSdkConfig()); + var materialProviders = new MaterialProviders(new MaterialProvidersConfig()); + + // A generator key plus optional additional keys. The first key acts as the generator. + if (keyIds.Length == 1) + { + _keyring = materialProviders.CreateAwsKmsKeyring(new CreateAwsKmsKeyringInput + { + KmsClient = kmsClient, + KmsKeyId = keyIds[0] + }); + } + else + { + // The generator is the first key; the remaining keys are additional child keys. + var additionalKeys = new List(keyIds.Length - 1); + for (var i = 1; i < keyIds.Length; i++) + { + additionalKeys.Add(keyIds[i]); + } + + _keyring = materialProviders.CreateAwsKmsMultiKeyring(new CreateAwsKmsMultiKeyringInput + { + Generator = keyIds[0], + KmsKeyIds = additionalKeys + }); + } + } + + /// + public Task EncryptAsync( + string value, + IDictionary? encryptionContext = null, + CancellationToken cancellationToken = default) + { + if (value is null) throw new ArgumentNullException(nameof(value)); + cancellationToken.ThrowIfCancellationRequested(); + + var encryptInput = new EncryptInput + { + Plaintext = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(value)), + Keyring = _keyring + }; + + if (encryptionContext is { Count: > 0 }) + { + encryptInput.EncryptionContext = new Dictionary(encryptionContext); + } + + var output = _encryptionSdk.Encrypt(encryptInput); + var ciphertext = ToArray(output.Ciphertext); + return Task.FromResult(Convert.ToBase64String(ciphertext)); + } + + /// + public Task DecryptAsync( + string value, + IDictionary? encryptionContext = null, + CancellationToken cancellationToken = default) + { + if (value is null) throw new ArgumentNullException(nameof(value)); + cancellationToken.ThrowIfCancellationRequested(); + + var decryptInput = new DecryptInput + { + Ciphertext = new MemoryStream(Convert.FromBase64String(value)), + Keyring = _keyring + }; + + if (encryptionContext is { Count: > 0 }) + { + decryptInput.EncryptionContext = new Dictionary(encryptionContext); + } + + var output = _encryptionSdk.Decrypt(decryptInput); + var plaintext = ToArray(output.Plaintext); + return Task.FromResult(System.Text.Encoding.UTF8.GetString(plaintext)); + } + + private static byte[] ToArray(MemoryStream stream) + { + return stream.ToArray(); + } +} diff --git a/libraries/src/AWS.Lambda.Powertools.DataMasking/Providers/IDataMaskingProvider.cs b/libraries/src/AWS.Lambda.Powertools.DataMasking/Providers/IDataMaskingProvider.cs new file mode 100644 index 00000000..4fdddbd1 --- /dev/null +++ b/libraries/src/AWS.Lambda.Powertools.DataMasking/Providers/IDataMaskingProvider.cs @@ -0,0 +1,59 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace AWS.Lambda.Powertools.DataMasking.Providers; + +/// +/// Abstraction for the encryption backend used by to encrypt and decrypt values. +/// +/// +/// The default implementation, , uses the AWS Encryption SDK for .NET +/// with AWS KMS envelope encryption. Implement this interface to provide a custom encryption backend. +/// +public interface IDataMaskingProvider +{ + /// + /// Encrypts a plaintext value. + /// + /// The plaintext value to encrypt. + /// + /// Optional non-secret key-value pairs bound to the ciphertext as additional authenticated data. + /// The same context must be supplied (and is validated) on decrypt. + /// + /// A token to cancel the operation. + /// The encrypted value, encoded as a string that can later be passed to . + Task EncryptAsync( + string value, + IDictionary? encryptionContext = null, + CancellationToken cancellationToken = default); + + /// + /// Decrypts a value previously produced by . + /// + /// The encrypted value. + /// + /// Optional encryption context to validate against the context bound at encrypt time. + /// + /// A token to cancel the operation. + /// The decrypted plaintext value. + Task DecryptAsync( + string value, + IDictionary? encryptionContext = null, + CancellationToken cancellationToken = default); +} diff --git a/libraries/src/AWS.Lambda.Powertools.DataMasking/README.md b/libraries/src/AWS.Lambda.Powertools.DataMasking/README.md index 66ae92f9..58e6f358 100644 --- a/libraries/src/AWS.Lambda.Powertools.DataMasking/README.md +++ b/libraries/src/AWS.Lambda.Powertools.DataMasking/README.md @@ -4,16 +4,19 @@ The Data Masking utility protects sensitive data (PII, credentials, financial re logged, stored, or forwarded to another service, helping to meet requirements such as GDPR, HIPAA, and PCI-DSS. -> **Status:** this package currently ships **Phase 1 – `Erase`** (irreversible masking). Field-level -> and full-payload encryption/decryption through a pluggable provider (AWS Encryption SDK + KMS) are -> planned for later phases. See [issue #1257](https://github.com/aws-powertools/powertools-lambda-dotnet/issues/1257). +> **Status:** this package ships **`Erase`** (irreversible masking) and **`Encrypt`/`Decrypt`** +> (reversible, via a pluggable provider using the AWS Encryption SDK + KMS). Logging integration and +> docs parity with other runtimes are planned next. +> See [issue #1257](https://github.com/aws-powertools/powertools-lambda-dotnet/issues/1257). ## Key features - Irreversibly **erase** sensitive fields while keeping the surrounding payload structure intact +- **Encrypt/decrypt** whole payloads or specific fields with AWS KMS envelope encryption +- Pluggable encryption provider (`IDataMaskingProvider`); default is `AwsEncryptionSdkProvider` - Select fields with simple dotted paths (for example `address.street`) - Customizable masking: fixed string, length-preserving mask, or regex + replacement -- Trimming/Native AOT friendly when operating over JSON strings or `JsonNode` +- `Erase` over JSON strings or `JsonNode` is trimming/Native AOT friendly ## Erase @@ -56,8 +59,39 @@ masker.Erase(json, new[] { "email" }, new MaskingOptions }); ``` +## Encrypt / Decrypt + +Unlike `Erase`, encryption is **reversible**. Construct `DataMasking` with an `IDataMaskingProvider`. +The default provider, `AwsEncryptionSdkProvider`, performs AWS KMS envelope encryption. + +```csharp +using AWS.Lambda.Powertools.DataMasking; +using AWS.Lambda.Powertools.DataMasking.Providers; + +var provider = new AwsEncryptionSdkProvider(KMS_KEY_ARN); +var masker = new DataMasking(provider); + +// Encrypt only specific fields; the rest of the payload stays readable +var encrypted = await masker.EncryptAsync(json, + fields: new[] { "customer.ssn", "payment.creditCard" }, + encryptionContext: new Dictionary { ["tenantId"] = "acme-corp" }); + +// Decrypt restores the original values (encryption context is validated) +var decrypted = await masker.DecryptAsync(encrypted, + fields: new[] { "customer.ssn", "payment.creditCard" }, + encryptionContext: new Dictionary { ["tenantId"] = "acme-corp" }); +``` + +You can also encrypt/decrypt the whole payload by omitting `fields`. The `encryptionContext` is bound +to the ciphertext at encrypt time and validated on decrypt, so a mismatched context fails. + ### AOT / trimming The `Erase(string, ...)` and `Erase(JsonNode, ...)` overloads are fully trimming/AOT safe. The `Erase(object, ...)` overload uses reflection-based serialization and is annotated accordingly; for AOT scenarios use the overload that accepts a source-generated `JsonTypeInfo`. + +The `AwsEncryptionSdkProvider` relies on the AWS Encryption SDK for .NET (Dafny-based runtime), which +uses reflection and is therefore **not** compatible with trimming/Native AOT. It is annotated with +`[RequiresUnreferencedCode]`/`[RequiresDynamicCode]`. If you need AOT, supply a custom +`IDataMaskingProvider` implementation that is AOT-compatible. diff --git a/libraries/src/Directory.Packages.props b/libraries/src/Directory.Packages.props index 665d5a3b..5c889bac 100644 --- a/libraries/src/Directory.Packages.props +++ b/libraries/src/Directory.Packages.props @@ -6,6 +6,8 @@ + + diff --git a/libraries/tests/AWS.Lambda.Powertools.DataMasking.Tests/EncryptDecryptTests.cs b/libraries/tests/AWS.Lambda.Powertools.DataMasking.Tests/EncryptDecryptTests.cs new file mode 100644 index 00000000..725a56fc --- /dev/null +++ b/libraries/tests/AWS.Lambda.Powertools.DataMasking.Tests/EncryptDecryptTests.cs @@ -0,0 +1,138 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using FluentAssertions; +using Xunit; + +namespace AWS.Lambda.Powertools.DataMasking.Tests; + +public class EncryptDecryptTests +{ + private const string SampleJson = """ + { + "orderId": "ORD-1001", + "customer": { "name": "John Doe", "ssn": "123-45-6789" }, + "payment": { "creditCard": "4111111111111111", "amount": 42.5 } + } + """; + + private static DataMasking CreateMasker() => new(new FakeDataMaskingProvider()); + + [Fact] + public async Task EncryptDecrypt_FullPayload_RoundTrips() + { + var masker = CreateMasker(); + + var encrypted = await masker.EncryptAsync(SampleJson); + encrypted.Should().NotContain("123-45-6789"); + + var decrypted = await masker.DecryptAsync(encrypted); + + // Compare structurally (formatting/whitespace independent) + var expected = JsonNode.Parse(SampleJson)!.ToJsonString(); + var actual = JsonNode.Parse(decrypted)!.ToJsonString(); + actual.Should().Be(expected); + } + + [Fact] + public async Task EncryptDecrypt_FieldLevel_LeavesOtherFieldsReadable() + { + var masker = CreateMasker(); + var fields = new[] { "customer.ssn", "payment.creditCard" }; + + var encrypted = await masker.EncryptAsync(SampleJson, fields); + + var node = JsonNode.Parse(encrypted)!; + // Selected fields are encrypted... + node["customer"]!["ssn"]!.GetValue().Should().StartWith("enc:"); + node["payment"]!["creditCard"]!.GetValue().Should().StartWith("enc:"); + // ...the rest stays readable + node["orderId"]!.GetValue().Should().Be("ORD-1001"); + node["customer"]!["name"]!.GetValue().Should().Be("John Doe"); + + var decrypted = await masker.DecryptAsync(encrypted, fields); + var restored = JsonNode.Parse(decrypted)!; + restored["customer"]!["ssn"]!.GetValue().Should().Be("123-45-6789"); + restored["payment"]!["creditCard"]!.GetValue().Should().Be("4111111111111111"); + } + + [Fact] + public async Task EncryptDecrypt_WithMatchingEncryptionContext_RoundTrips() + { + var masker = CreateMasker(); + var context = new Dictionary { ["tenantId"] = "acme-corp" }; + + var encrypted = await masker.EncryptAsync(SampleJson, context); + var decrypted = await masker.DecryptAsync(encrypted, context); + + JsonNode.Parse(decrypted)!.ToJsonString() + .Should().Be(JsonNode.Parse(SampleJson)!.ToJsonString()); + } + + [Fact] + public async Task Decrypt_WithMismatchedEncryptionContext_Throws() + { + var masker = CreateMasker(); + var encryptContext = new Dictionary { ["tenantId"] = "acme-corp" }; + var wrongContext = new Dictionary { ["tenantId"] = "evil-corp" }; + + var encrypted = await masker.EncryptAsync(SampleJson, encryptContext); + + var act = async () => await masker.DecryptAsync(encrypted, wrongContext); + await act.Should().ThrowAsync(); + } + + [Fact] + public async Task Encrypt_WithoutProvider_Throws() + { + var masker = new DataMasking(); // no provider + + var act = async () => await masker.EncryptAsync(SampleJson); + + await act.Should().ThrowAsync(); + } + + [Fact] + public async Task Decrypt_WithoutProvider_Throws() + { + var masker = new DataMasking(); // no provider + + var act = async () => await masker.DecryptAsync("enc:whatever"); + + await act.Should().ThrowAsync(); + } + + [Fact] + public void Constructor_NullProvider_Throws() + { + var act = () => new DataMasking(null!); + + act.Should().Throw(); + } + + [Fact] + public async Task EncryptAsync_NullJson_Throws() + { + var masker = CreateMasker(); + + var act = async () => await masker.EncryptAsync((string)null!); + + await act.Should().ThrowAsync(); + } +} diff --git a/libraries/tests/AWS.Lambda.Powertools.DataMasking.Tests/FakeDataMaskingProvider.cs b/libraries/tests/AWS.Lambda.Powertools.DataMasking.Tests/FakeDataMaskingProvider.cs new file mode 100644 index 00000000..8b52c831 --- /dev/null +++ b/libraries/tests/AWS.Lambda.Powertools.DataMasking.Tests/FakeDataMaskingProvider.cs @@ -0,0 +1,84 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using AWS.Lambda.Powertools.DataMasking.Providers; + +namespace AWS.Lambda.Powertools.DataMasking.Tests; + +/// +/// A deterministic, in-memory used to exercise the DataMasking +/// Encrypt/Decrypt orchestration without depending on AWS KMS. It base64-encodes with a fixed prefix +/// and, when an encryption context is supplied, binds it so that a mismatched context fails on decrypt. +/// +internal sealed class FakeDataMaskingProvider : IDataMaskingProvider +{ + private const string Prefix = "enc:"; + + public Task EncryptAsync( + string value, + IDictionary? encryptionContext = null, + CancellationToken cancellationToken = default) + { + var context = SerializeContext(encryptionContext); + var payload = $"{context}|{value}"; + var encoded = Prefix + Convert.ToBase64String(Encoding.UTF8.GetBytes(payload)); + return Task.FromResult(encoded); + } + + public Task DecryptAsync( + string value, + IDictionary? encryptionContext = null, + CancellationToken cancellationToken = default) + { + if (!value.StartsWith(Prefix, StringComparison.Ordinal)) + { + throw new InvalidOperationException("Value was not produced by this provider."); + } + + var decoded = Encoding.UTF8.GetString(Convert.FromBase64String(value.Substring(Prefix.Length))); + var separator = decoded.IndexOf('|'); + var boundContext = decoded.Substring(0, separator); + var plaintext = decoded.Substring(separator + 1); + + if (boundContext != SerializeContext(encryptionContext)) + { + throw new InvalidOperationException("Encryption context mismatch."); + } + + return Task.FromResult(plaintext); + } + + private static string SerializeContext(IDictionary? context) + { + if (context is null || context.Count == 0) + { + return string.Empty; + } + + var parts = new List(context.Count); + foreach (var kvp in context) + { + parts.Add($"{kvp.Key}={kvp.Value}"); + } + + parts.Sort(StringComparer.Ordinal); + return string.Join(";", parts); + } +} From 5a2b2a54f4d6309f34a10524e43b2b1141a58801 Mon Sep 17 00:00:00 2001 From: Nicolas Tarzia Date: Thu, 24 Sep 2026 00:04:31 -0300 Subject: [PATCH 3/5] feat(data-masking): add Logging integration via EraseToNode (Phase 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an ergonomic way to redact sensitive fields before they reach logs, without coupling the Data Masking and Logging packages. - DataMasking.EraseToNode(string json, string[] fields, MaskingOptions?) returns a JsonNode ready for structured logging, e.g. Logger.LogInformation(masker.EraseToNode(json, fields)) — no extra string round-trip. Fully trimming/AOT safe. - Example updated to log the masked payload via EraseToNode - Package and example READMEs updated with the Logging integration Relates to #1257. Phase 3 (Logging integration + docs) completing the Erase / Encrypt / Decrypt trio. 25 unit tests pass on net10.0. --- examples/DataMasking/README.md | 8 +- .../DataMasking/src/HelloWorld/Function.cs | 13 ++- .../DataMasking.cs | 21 +++++ .../README.md | 30 ++++++- .../EraseToNodeTests.cs | 81 +++++++++++++++++++ 5 files changed, 146 insertions(+), 7 deletions(-) create mode 100644 libraries/tests/AWS.Lambda.Powertools.DataMasking.Tests/EraseToNodeTests.cs diff --git a/examples/DataMasking/README.md b/examples/DataMasking/README.md index bbc2d2f9..e6821279 100644 --- a/examples/DataMasking/README.md +++ b/examples/DataMasking/README.md @@ -4,8 +4,8 @@ This example shows how to use the [Data Masking](https://github.com/aws-powertoo utility to irreversibly **erase** sensitive fields (PII) from a payload before it is logged or returned, while keeping the non-sensitive fields readable. -> This example covers **Phase 1 (`Erase`)** of the utility. Field-level and full-payload -> encryption/decryption via a pluggable provider (AWS Encryption SDK + KMS) are planned for later phases. +This example focuses on the `Erase` operation and its integration with Powertools Logging. For +reversible `Encrypt`/`Decrypt` with AWS KMS, see the utility README. ## How it works @@ -20,6 +20,10 @@ The result keeps `orderId`, `item`, `payment.amount`, and `address.city` readabl remains useful for debugging and monitoring. `LogEvent` is intentionally disabled so the raw PII is never auto-logged. +**Logging integration:** the handler logs the masked payload with +`Logger.LogInformation(_dataMasking.EraseToNode(...))`. `EraseToNode` returns a `JsonNode`, so the +masked object is logged as structured JSON with no extra string round-trip. + ## Prerequisites - .NET 8.0 - [Install .NET 8.0](https://www.microsoft.com/net/download) diff --git a/examples/DataMasking/src/HelloWorld/Function.cs b/examples/DataMasking/src/HelloWorld/Function.cs index 7d50b6af..ecbf26f7 100644 --- a/examples/DataMasking/src/HelloWorld/Function.cs +++ b/examples/DataMasking/src/HelloWorld/Function.cs @@ -61,8 +61,17 @@ public APIGatewayProxyResponse FunctionHandler(APIGatewayProxyRequest request, I PreserveLength = true }); - // Safe to log now: sensitive fields are erased, structure is intact - Logger.LogInformation("Processing masked order: {Order}", masked); + // Logging integration: EraseToNode returns a JsonNode ready for structured logging, so the + // masked payload is logged as a JSON object (not a string) with no extra serialization round-trip. + // Sensitive fields are already erased, so this is safe to log. + Logger.LogInformation(_dataMasking.EraseToNode(body, new[] + { + "customer.ssn", + "customer.email", + "customer.phone", + "payment.creditCard", + "address.street" + })); return new APIGatewayProxyResponse { diff --git a/libraries/src/AWS.Lambda.Powertools.DataMasking/DataMasking.cs b/libraries/src/AWS.Lambda.Powertools.DataMasking/DataMasking.cs index 6c0e0edd..0469e159 100644 --- a/libraries/src/AWS.Lambda.Powertools.DataMasking/DataMasking.cs +++ b/libraries/src/AWS.Lambda.Powertools.DataMasking/DataMasking.cs @@ -57,6 +57,27 @@ public DataMasking(IDataMaskingProvider provider) _provider = provider ?? throw new ArgumentNullException(nameof(provider)); } + /// + /// Irreversibly masks the supplied fields within a JSON string and returns the result as a + /// , ready to be passed to structured logging (for example + /// Logger.LogInformation(masker.EraseToNode(json, fields))) without an extra string round-trip. + /// This overload is fully trimming/AOT safe. + /// + /// The JSON payload as a string. + /// The dotted field paths to mask. + /// Optional masking options. Defaults to a fixed ***** mask. + /// The masked payload as a (or if the payload was JSON null). + /// or is . + public JsonNode? EraseToNode(string json, string[] fields, MaskingOptions? options = null) + { + if (json is null) throw new ArgumentNullException(nameof(json)); + if (fields is null) throw new ArgumentNullException(nameof(fields)); + + var root = JsonNode.Parse(json); + JsonNodeMasker.MaskFields(root, fields, options ?? MaskingOptions.Default); + return root; + } + /// /// Irreversibly masks the supplied fields within a JSON string. /// This overload is fully trimming/AOT safe. diff --git a/libraries/src/AWS.Lambda.Powertools.DataMasking/README.md b/libraries/src/AWS.Lambda.Powertools.DataMasking/README.md index 58e6f358..a575cfad 100644 --- a/libraries/src/AWS.Lambda.Powertools.DataMasking/README.md +++ b/libraries/src/AWS.Lambda.Powertools.DataMasking/README.md @@ -85,11 +85,35 @@ var decrypted = await masker.DecryptAsync(encrypted, You can also encrypt/decrypt the whole payload by omitting `fields`. The `encryptionContext` is bound to the ciphertext at encrypt time and validated on decrypt, so a mismatched context fails. +## Logging integration + +To redact sensitive fields before they reach your logs, mask the payload and log the result. +`EraseToNode` returns a `JsonNode` that Powertools Logging serializes directly, avoiding an extra +string round-trip: + +```csharp +using AWS.Lambda.Powertools.DataMasking; +using AWS.Lambda.Powertools.Logging; + +var masker = new DataMasking(); + +Logger.LogInformation(masker.EraseToNode(json, new[] { "customer.ssn", "payment.creditCard" })); +``` + +Data Masking and Logging stay independent packages: the integration is just this pattern, so you can +mask before logging without coupling the two utilities. + ### AOT / trimming -The `Erase(string, ...)` and `Erase(JsonNode, ...)` overloads are fully trimming/AOT safe. The -`Erase(object, ...)` overload uses reflection-based serialization and is annotated accordingly; for -AOT scenarios use the overload that accepts a source-generated `JsonTypeInfo`. +The `Erase(string, ...)`, `Erase(JsonNode, ...)`, and `EraseToNode(...)` overloads are fully +trimming/AOT safe. The `Erase(object, ...)` overload uses reflection-based serialization and is +annotated accordingly; for AOT scenarios use the overload that accepts a source-generated +`JsonTypeInfo`. + +The `AwsEncryptionSdkProvider` relies on the AWS Encryption SDK for .NET (Dafny-based runtime), which +uses reflection and is therefore **not** compatible with trimming/Native AOT. It is annotated with +`[RequiresUnreferencedCode]`/`[RequiresDynamicCode]`. If you need AOT, supply a custom +`IDataMaskingProvider` implementation that is AOT-compatible. The `AwsEncryptionSdkProvider` relies on the AWS Encryption SDK for .NET (Dafny-based runtime), which uses reflection and is therefore **not** compatible with trimming/Native AOT. It is annotated with diff --git a/libraries/tests/AWS.Lambda.Powertools.DataMasking.Tests/EraseToNodeTests.cs b/libraries/tests/AWS.Lambda.Powertools.DataMasking.Tests/EraseToNodeTests.cs new file mode 100644 index 00000000..27323fee --- /dev/null +++ b/libraries/tests/AWS.Lambda.Powertools.DataMasking.Tests/EraseToNodeTests.cs @@ -0,0 +1,81 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +using System; +using System.Text.Json.Nodes; +using FluentAssertions; +using Xunit; + +namespace AWS.Lambda.Powertools.DataMasking.Tests; + +/// +/// Tests for the logging-oriented convenience that returns a suitable for +/// passing directly to Powertools Logging (Logger.LogInformation(node)). +/// +public class EraseToNodeTests +{ + private const string SampleJson = """ + { + "orderId": "ORD-1001", + "customer": { "name": "John Doe", "ssn": "123-45-6789" } + } + """; + + [Fact] + public void EraseToNode_ReturnsMaskedNode_WithSensitiveFieldErased() + { + var masker = new DataMasking(); + + var node = masker.EraseToNode(SampleJson, new[] { "customer.ssn" }); + + node.Should().NotBeNull(); + node!["customer"]!["ssn"]!.GetValue().Should().Be("*****"); + node["customer"]!["name"]!.GetValue().Should().Be("John Doe"); + node["orderId"]!.GetValue().Should().Be("ORD-1001"); + } + + [Fact] + public void EraseToNode_ResultIsSerializable_ForStructuredLogging() + { + var masker = new DataMasking(); + + var node = masker.EraseToNode(SampleJson, new[] { "customer.ssn" }); + + // Simulates what the logger does: serialize the object. Must not throw and must not leak PII. + var serialized = node!.ToJsonString(); + serialized.Should().NotContain("123-45-6789"); + serialized.Should().Contain("*****"); + } + + [Fact] + public void EraseToNode_NullJson_Throws() + { + var masker = new DataMasking(); + + var act = () => masker.EraseToNode(null!, new[] { "customer.ssn" }); + + act.Should().Throw(); + } + + [Fact] + public void EraseToNode_NullFields_Throws() + { + var masker = new DataMasking(); + + var act = () => masker.EraseToNode(SampleJson, null!); + + act.Should().Throw(); + } +} From 68d7fc09b65c77ce6ad8b42cb472ecdfa70338e5 Mon Sep 17 00:00:00 2001 From: Nicolas Tarzia Date: Thu, 24 Sep 2026 00:29:44 -0300 Subject: [PATCH 4/5] fix(data-masking): address SonarCloud quality gate findings - Add a timeout to the example Regex to avoid potential ReDoS (fixes the Security rating on new code) - Reduce code duplication in DataMasking by extracting MaskRoot/Serialize/ TransformFieldsToStringAsync helpers (fixes duplication > 3% on new code) - Declare an explicit CloudWatch Log Group in the example SAM template (with 7-day retention), matching the BatchProcessing example All 25 unit tests still pass on net10.0. --- .../DataMasking/src/HelloWorld/Function.cs | 3 +- examples/DataMasking/template.yaml | 6 +++ .../DataMasking.cs | 40 ++++++++++--------- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/examples/DataMasking/src/HelloWorld/Function.cs b/examples/DataMasking/src/HelloWorld/Function.cs index ecbf26f7..bc8482e1 100644 --- a/examples/DataMasking/src/HelloWorld/Function.cs +++ b/examples/DataMasking/src/HelloWorld/Function.cs @@ -13,6 +13,7 @@ * permissions and limitations under the License. */ +using System; using System.Collections.Generic; using System.Text.RegularExpressions; using Amazon.Lambda.APIGatewayEvents; @@ -52,7 +53,7 @@ public APIGatewayProxyResponse FunctionHandler(APIGatewayProxyRequest request, I // 2. Custom rules: keep the first char of the email, and preserve the phone length masked = _dataMasking.Erase(masked, new[] { "customer.email" }, new MaskingOptions { - Pattern = new Regex("^(.).*@"), + Pattern = new Regex("^(.).*@", RegexOptions.None, TimeSpan.FromSeconds(1)), Replacement = "$1****@" }); diff --git a/examples/DataMasking/template.yaml b/examples/DataMasking/template.yaml index c22cb2e1..c70854ee 100644 --- a/examples/DataMasking/template.yaml +++ b/examples/DataMasking/template.yaml @@ -29,6 +29,12 @@ Resources: Path: /mask Method: post + HelloWorldFunctionLogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub "/aws/lambda/${HelloWorldFunction}" + RetentionInDays: 7 + Outputs: # ServerlessRestApi is an implicit API created out of Events key under Serverless::Function # Find out more about other implicit resources you can reference within SAM diff --git a/libraries/src/AWS.Lambda.Powertools.DataMasking/DataMasking.cs b/libraries/src/AWS.Lambda.Powertools.DataMasking/DataMasking.cs index 0469e159..9b140f7e 100644 --- a/libraries/src/AWS.Lambda.Powertools.DataMasking/DataMasking.cs +++ b/libraries/src/AWS.Lambda.Powertools.DataMasking/DataMasking.cs @@ -73,9 +73,7 @@ public DataMasking(IDataMaskingProvider provider) if (json is null) throw new ArgumentNullException(nameof(json)); if (fields is null) throw new ArgumentNullException(nameof(fields)); - var root = JsonNode.Parse(json); - JsonNodeMasker.MaskFields(root, fields, options ?? MaskingOptions.Default); - return root; + return MaskRoot(JsonNode.Parse(json), fields, options); } /// @@ -92,9 +90,7 @@ public string Erase(string json, string[] fields, MaskingOptions? options = null if (json is null) throw new ArgumentNullException(nameof(json)); if (fields is null) throw new ArgumentNullException(nameof(fields)); - var root = JsonNode.Parse(json); - JsonNodeMasker.MaskFields(root, fields, options ?? MaskingOptions.Default); - return root?.ToJsonString() ?? "null"; + return Serialize(MaskRoot(JsonNode.Parse(json), fields, options)); } /// @@ -137,9 +133,7 @@ public string Erase(object data, string[] fields, MaskingOptions? options = null if (data is null) throw new ArgumentNullException(nameof(data)); if (fields is null) throw new ArgumentNullException(nameof(fields)); - var root = JsonSerializer.SerializeToNode(data); - JsonNodeMasker.MaskFields(root, fields, options ?? MaskingOptions.Default); - return root?.ToJsonString() ?? "null"; + return Serialize(MaskRoot(JsonSerializer.SerializeToNode(data), fields, options)); } /// @@ -159,9 +153,7 @@ public string Erase(object data, JsonTypeInfo jsonTypeInfo, string[] fields, Mas if (jsonTypeInfo is null) throw new ArgumentNullException(nameof(jsonTypeInfo)); if (fields is null) throw new ArgumentNullException(nameof(fields)); - var root = JsonSerializer.SerializeToNode(data, jsonTypeInfo); - JsonNodeMasker.MaskFields(root, fields, options ?? MaskingOptions.Default); - return root?.ToJsonString() ?? "null"; + return Serialize(MaskRoot(JsonSerializer.SerializeToNode(data, jsonTypeInfo), fields, options)); } /// @@ -202,10 +194,8 @@ public async Task EncryptAsync( if (fields is null) throw new ArgumentNullException(nameof(fields)); var provider = RequireProvider(); - var root = JsonNode.Parse(json); - await JsonNodeMasker.TransformFieldsAsync(root, fields, + return await TransformFieldsToStringAsync(json, fields, value => provider.EncryptAsync(value, encryptionContext, cancellationToken)); - return root?.ToJsonString() ?? "null"; } /// @@ -247,10 +237,8 @@ public async Task DecryptAsync( if (fields is null) throw new ArgumentNullException(nameof(fields)); var provider = RequireProvider(); - var root = JsonNode.Parse(json); - await JsonNodeMasker.TransformFieldsAsync(root, fields, + return await TransformFieldsToStringAsync(json, fields, value => provider.DecryptAsync(value, encryptionContext, cancellationToken)); - return root?.ToJsonString() ?? "null"; } private IDataMaskingProvider RequireProvider() @@ -258,4 +246,20 @@ private IDataMaskingProvider RequireProvider() return _provider ?? throw new InvalidOperationException( "This operation requires an encryption provider. Construct DataMasking with an IDataMaskingProvider (for example new DataMasking(new AwsEncryptionSdkProvider(keyArn)))."); } + + private static JsonNode? MaskRoot(JsonNode? root, string[] fields, MaskingOptions? options) + { + JsonNodeMasker.MaskFields(root, fields, options ?? MaskingOptions.Default); + return root; + } + + private static string Serialize(JsonNode? root) => root?.ToJsonString() ?? "null"; + + private static async Task TransformFieldsToStringAsync( + string json, string[] fields, Func> transform) + { + var root = JsonNode.Parse(json); + await JsonNodeMasker.TransformFieldsAsync(root, fields, transform); + return Serialize(root); + } } From 95fe44734ec18725deef3709a591886965fcbd52 Mon Sep 17 00:00:00 2001 From: Nicolas Tarzia Date: Thu, 24 Sep 2026 00:33:28 -0300 Subject: [PATCH 5/5] refactor(data-masking): dedupe JsonNode path walking Extract a shared TryResolveLeaf (and ReadRawValue) helper so the sync mask and async transform paths no longer duplicate the dotted-path walk, bringing new-code duplication under the SonarCloud threshold. Behavior unchanged; 25 tests still pass. --- .../Internal/JsonNodeMasker.cs | 81 ++++++++++--------- 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/libraries/src/AWS.Lambda.Powertools.DataMasking/Internal/JsonNodeMasker.cs b/libraries/src/AWS.Lambda.Powertools.DataMasking/Internal/JsonNodeMasker.cs index 04e48e7e..2c917370 100644 --- a/libraries/src/AWS.Lambda.Powertools.DataMasking/Internal/JsonNodeMasker.cs +++ b/libraries/src/AWS.Lambda.Powertools.DataMasking/Internal/JsonNodeMasker.cs @@ -59,9 +59,27 @@ internal static void MaskFields(JsonNode? root, IEnumerable fields, Mask private static void MaskPath(JsonNode root, string path, MaskingOptions options) { - // Walk the dotted path segment-by-segment without allocating the intermediate string[] - // that string.Split would produce. JsonObject lookups require a string key, so each - // segment is still materialized, but the array (and its bounds/GC overhead) is avoided. + if (TryResolveLeaf(root, path, out var parent, out var leaf)) + { + parent[leaf] = MaskValue(parent[leaf], options); + } + } + + /// + /// Walks the dotted and, if it resolves to a property on an object, + /// returns that object and the leaf key. Returns when any intermediate + /// segment is missing or is not an object, or when the leaf property does not exist. + /// + /// + /// The walk avoids the intermediate string[] that string.Split would allocate. + /// JsonObject lookups require a string key, so each segment is still materialized, but the array + /// (and its bounds/GC overhead) is avoided. + /// + private static bool TryResolveLeaf(JsonNode root, string path, out JsonObject parent, out string leaf) + { + parent = null!; + leaf = string.Empty; + var current = root; var start = 0; @@ -69,28 +87,34 @@ private static void MaskPath(JsonNode root, string path, MaskingOptions options) while ((separator = path.IndexOf(PathSeparator, start)) >= 0) { var segment = path.Substring(start, separator - start); - - // If any intermediate segment is missing or is not an object, the path does not - // exist in this payload and is skipped. if (current is not JsonObject obj || !obj.TryGetPropertyValue(segment, out var next) || next is null) { - return; + return false; } current = next; start = separator + 1; } - if (current is JsonObject parent) + if (current is not JsonObject leafParent) { - var leaf = start == 0 ? path : path.Substring(start); - if (parent.ContainsKey(leaf)) - { - parent[leaf] = MaskValue(parent[leaf], options); - } + return false; + } + + var key = start == 0 ? path : path.Substring(start); + if (!leafParent.ContainsKey(key)) + { + return false; } + + parent = leafParent; + leaf = key; + return true; } + private static string ReadRawValue(JsonNode value) => + value is JsonValue jsonValue ? jsonValue.ToString() : value.ToJsonString(); + private static JsonNode? MaskValue(JsonNode? value, MaskingOptions options) { // A null JSON value stays null; there is nothing sensitive to mask. @@ -106,8 +130,7 @@ private static void MaskPath(JsonNode root, string path, MaskingOptions options) return JsonValue.Create(options.Apply(string.Empty)); } - var raw = value is JsonValue jsonValue ? jsonValue.ToString() : value.ToJsonString(); - return JsonValue.Create(options.Apply(raw)); + return JsonValue.Create(options.Apply(ReadRawValue(value))); } /// @@ -137,32 +160,10 @@ internal static async Task TransformFieldsAsync( private static async Task TransformPathAsync(JsonNode root, string path, Func> transform) { - var current = root; - var start = 0; - - int separator; - while ((separator = path.IndexOf(PathSeparator, start)) >= 0) - { - var segment = path.Substring(start, separator - start); - if (current is not JsonObject obj || !obj.TryGetPropertyValue(segment, out var next) || next is null) - { - return; - } - - current = next; - start = separator + 1; - } - - if (current is JsonObject parent) + if (TryResolveLeaf(root, path, out var parent, out var leaf) && parent[leaf] is not null) { - var leaf = start == 0 ? path : path.Substring(start); - if (parent.ContainsKey(leaf) && parent[leaf] is not null) - { - var value = parent[leaf]!; - var raw = value is JsonValue jsonValue ? jsonValue.ToString() : value.ToJsonString(); - var transformed = await transform(raw); - parent[leaf] = JsonValue.Create(transformed); - } + var transformed = await transform(ReadRawValue(parent[leaf]!)); + parent[leaf] = JsonValue.Create(transformed); } } }