From b37f00d78a439dd9154ce95cefa3633a112037d5 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Fri, 16 Oct 2026 10:00:56 +0800 Subject: [PATCH 1/2] Enhance JSON serialization options and add tests for email address handling in message headers --- .../Infrastructure/MessageHeaders.cs | 11 +++++++++-- .../EFCore/EFQueryTests.cs | 12 ++++++++++++ .../Infrastructure/MessageHeaders.cs | 11 +++++++++-- .../EFCore/MessagesViewDataStoreTests.cs | 16 +++++++++++++++- 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/MessageHeaders.cs b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/MessageHeaders.cs index 4ead155690..0ba418382c 100644 --- a/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/MessageHeaders.cs +++ b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/MessageHeaders.cs @@ -12,8 +12,15 @@ public static string Write(Dictionary headers) => public static Dictionary Read(string headersJson) => JsonSerializer.Deserialize(headersJson, context.DictionaryStringString) ?? []; - // The relaxed encoder is only unsafe for JSON embedded in HTML, and this JSON never reaches HTML. Full text search needs apostrophes, plus signs and non-ASCII letters written as they are, or it cannot find the words next to them. - static readonly HeadersJsonContext context = new(new JsonSerializerOptions { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }); + static readonly HeadersJsonContext context = new(new JsonSerializerOptions + { + // The relaxed encoder is only unsafe for JSON embedded in HTML, and this JSON never reaches HTML. Full text search needs apostrophes, plus signs and non-ASCII letters written as they are, or it cannot find the words next to them. + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + + // Without whitespace between the entries, the SQL Server word breaker keeps JSON quotes on the words near an email address or a URL. A search for those words then finds nothing. + WriteIndented = true, + IndentSize = 0 + }); } [JsonSerializable(typeof(Dictionary))] diff --git a/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFQueryTests.cs b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFQueryTests.cs index ed284acaa3..24afbb9f51 100644 --- a/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFQueryTests.cs +++ b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFQueryTests.cs @@ -92,6 +92,18 @@ public async Task Does_not_match_messages_that_share_part_of_the_message_id() Assert.That(found.Results.Select(view => view.MessageId), Is.EqualTo(new[] { "billing-1234-5678" })); } + // The SQL Server word breaker keeps JSON quotes on the words near an email address unless the header entries are spaced apart. + // It only reaches a few hundred characters, so the content type header is left out to keep the address close to the id. + [Test] + public async Task Finds_the_message_id_when_a_header_after_it_looks_like_an_email_address() + { + await Ingest(MakeMessage(messageId: "billing-3333-5678", contentType: null, extraHeaders: new Dictionary { ["NServiceBus.ReplyToAddress"] = "Sales@web01.corp.example.com" })); + + var found = await MessagesViewStore.QueryMessages("billing-3333-5678", new PagingInfo(), new SortInfo("time_sent", "desc")); + + Assert.That(found.Results.Select(view => view.MessageId), Is.EqualTo(new[] { "billing-3333-5678" })); + } + [Test] public async Task Matches_a_term_ending_in_a_star_as_a_prefix() { diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/MessageHeaders.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/MessageHeaders.cs index fc3276dea5..298e365eba 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/MessageHeaders.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/MessageHeaders.cs @@ -13,8 +13,15 @@ public static string Write(Dictionary headers) => public static Dictionary Read(string headersJson) => JsonSerializer.Deserialize(headersJson, context.DictionaryStringString) ?? []; - // The relaxed encoder is only unsafe for JSON embedded in HTML, and this JSON never reaches HTML. Full text search needs apostrophes, plus signs and non-ASCII letters written as they are, or it cannot find the words next to them. - static readonly HeadersJsonContext context = new(new JsonSerializerOptions { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }); + static readonly HeadersJsonContext context = new(new JsonSerializerOptions + { + // The relaxed encoder is only unsafe for JSON embedded in HTML, and this JSON never reaches HTML. Full text search needs apostrophes, plus signs and non-ASCII letters written as they are, or it cannot find the words next to them. + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + + // Without whitespace between the entries, the SQL Server word breaker keeps JSON quotes on the words near an email address or a URL. A search for those words then finds nothing. + WriteIndented = true, + IndentSize = 0 + }); } // Source generated serialization, which keeps the reflection-based serializer off the ingestion hot path. diff --git a/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs index bbf1a75e32..539cbe8f9a 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs @@ -37,7 +37,7 @@ public async Task Reports_a_stored_failure() Assert.That(view.BodyUrl, Is.EqualTo($"/messages/{failure.UniqueMessageIdString}/body")); Assert.That(view.IsSystemMessage, Is.False); Assert.That(view.Headers.ToDictionary(header => header.Key, header => header.Value), - Does.ContainKey(NServiceBus.Headers.EnclosedMessageTypes)); + Does.ContainKey(Headers.EnclosedMessageTypes)); } } @@ -369,6 +369,20 @@ public async Task Searches_the_message_id() await AssertSearchFinds(matching.MessageId, matching); } + /// + /// The SQL Server word breaker keeps JSON quotes and colons on the words near an email address or a URL. + /// It only does this when nothing separates the header entries, and then the id cannot be found. + /// + [Test] + public async Task Searches_the_message_id_when_a_header_after_it_looks_like_an_email_address() + { + var matching = new IngestedFailure { QueueAddress = "error@web01.corp.example.com" }; + + await Ingest(matching); + + await AssertSearchFinds(matching.MessageId, matching); + } + [Test] public async Task Ors_the_search_terms() { From d521022e8d83502c25d052057a5679f7d7c07f98 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Fri, 16 Oct 2026 10:52:47 +0800 Subject: [PATCH 2/2] Refactor JSON handling and update tests for email address proximity in SQL Server --- .../Infrastructure/MessageHeaders.cs | 2 +- .../EFCore/EFQueryTests.cs | 20 +++++++++++--- .../Infrastructure/MessageHeaders.cs | 2 +- .../EFCore/MessagesViewDataStoreTests.cs | 10 +++---- .../IngestedFailure.cs | 26 ++++++++++++------- 5 files changed, 40 insertions(+), 20 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/MessageHeaders.cs b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/MessageHeaders.cs index 0ba418382c..394b79ca67 100644 --- a/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/MessageHeaders.cs +++ b/src/ServiceControl.Audit.Persistence.EFCore/Infrastructure/MessageHeaders.cs @@ -17,7 +17,7 @@ public static Dictionary Read(string headersJson) => // The relaxed encoder is only unsafe for JSON embedded in HTML, and this JSON never reaches HTML. Full text search needs apostrophes, plus signs and non-ASCII letters written as they are, or it cannot find the words next to them. Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, - // Without whitespace between the entries, the SQL Server word breaker keeps JSON quotes on the words near an email address or a URL. A search for those words then finds nothing. + // In compact JSON, SQL Server's word breaker keeps JSON quotes and colons on the words before an email address or a URL, and a search cannot find those words. An indent of 0 adds only the whitespace that stops this: a line break after each entry and a space after each colon. WriteIndented = true, IndentSize = 0 }); diff --git a/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFQueryTests.cs b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFQueryTests.cs index 24afbb9f51..2dada44e34 100644 --- a/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFQueryTests.cs +++ b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFQueryTests.cs @@ -92,12 +92,26 @@ public async Task Does_not_match_messages_that_share_part_of_the_message_id() Assert.That(found.Results.Select(view => view.MessageId), Is.EqualTo(new[] { "billing-1234-5678" })); } - // The SQL Server word breaker keeps JSON quotes on the words near an email address unless the header entries are spaced apart. - // It only reaches a few hundred characters, so the content type header is left out to keep the address close to the id. + // In compact JSON, SQL Server's word breaker keeps JSON quotes and colons on words up to a few hundred characters before an email address. + // The reply-to address goes straight after the id, so the test does not depend on how many headers MakeMessage writes. [Test] public async Task Finds_the_message_id_when_a_header_after_it_looks_like_an_email_address() { - await Ingest(MakeMessage(messageId: "billing-3333-5678", contentType: null, extraHeaders: new Dictionary { ["NServiceBus.ReplyToAddress"] = "Sales@web01.corp.example.com" })); + var message = MakeMessage(messageId: "billing-3333-5678"); + var headers = new Dictionary + { + [Headers.MessageId] = "billing-3333-5678", + [Headers.ReplyToAddress] = "Sales@web01.corp.example.com" + }; + + foreach (var header in message.Headers) + { + headers.TryAdd(header.Key, header.Value); + } + + message.Headers = headers; + + await Ingest(message); var found = await MessagesViewStore.QueryMessages("billing-3333-5678", new PagingInfo(), new SortInfo("time_sent", "desc")); diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/MessageHeaders.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/MessageHeaders.cs index 298e365eba..fcf7cf2ac7 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/MessageHeaders.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/MessageHeaders.cs @@ -18,7 +18,7 @@ public static Dictionary Read(string headersJson) => // The relaxed encoder is only unsafe for JSON embedded in HTML, and this JSON never reaches HTML. Full text search needs apostrophes, plus signs and non-ASCII letters written as they are, or it cannot find the words next to them. Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, - // Without whitespace between the entries, the SQL Server word breaker keeps JSON quotes on the words near an email address or a URL. A search for those words then finds nothing. + // In compact JSON, SQL Server's word breaker keeps JSON quotes and colons on the words before an email address or a URL, and a search cannot find those words. An indent of 0 adds only the whitespace that stops this: a line break after each entry and a space after each colon. WriteIndented = true, IndentSize = 0 }); diff --git a/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs index 539cbe8f9a..693dc85218 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs @@ -37,7 +37,7 @@ public async Task Reports_a_stored_failure() Assert.That(view.BodyUrl, Is.EqualTo($"/messages/{failure.UniqueMessageIdString}/body")); Assert.That(view.IsSystemMessage, Is.False); Assert.That(view.Headers.ToDictionary(header => header.Key, header => header.Value), - Does.ContainKey(Headers.EnclosedMessageTypes)); + Does.ContainKey(NServiceBus.Headers.EnclosedMessageTypes)); } } @@ -370,13 +370,13 @@ public async Task Searches_the_message_id() } /// - /// The SQL Server word breaker keeps JSON quotes and colons on the words near an email address or a URL. - /// It only does this when nothing separates the header entries, and then the id cannot be found. + /// In compact JSON, SQL Server's word breaker keeps JSON quotes and colons on words up to a few hundred characters before an email address. + /// The reply-to address goes straight after the id, so the test does not depend on how many other headers there are. /// [Test] - public async Task Searches_the_message_id_when_a_header_after_it_looks_like_an_email_address() + public async Task Finds_the_message_id_when_a_header_after_it_looks_like_an_email_address() { - var matching = new IngestedFailure { QueueAddress = "error@web01.corp.example.com" }; + var matching = new IngestedFailure { ReplyToAddress = "Sales@web01.corp.example.com" }; await Ingest(matching); diff --git a/src/ServiceControl.Persistence.Tests/IngestedFailure.cs b/src/ServiceControl.Persistence.Tests/IngestedFailure.cs index 9f17cf7e6d..ed92c7bbeb 100644 --- a/src/ServiceControl.Persistence.Tests/IngestedFailure.cs +++ b/src/ServiceControl.Persistence.Tests/IngestedFailure.cs @@ -24,6 +24,7 @@ class IngestedFailure public string MessageType { get; init; } = "MyCompany.Sales.OrderPlaced"; public string ConversationId { get; init; } = Guid.NewGuid().ToString(); public string QueueAddress { get; init; } = "error"; + public string ReplyToAddress { get; init; } public string FailingEndpointAddress { get; init; } public string ExceptionType { get; init; } = "System.InvalidOperationException"; public string ExceptionMessage { get; init; } = "Something went wrong"; @@ -42,17 +43,21 @@ class IngestedFailure Dictionary BuildHeaders() { - var headers = new Dictionary + var headers = new Dictionary { [NServiceBus.Headers.MessageId] = MessageId }; + + // Straight after the id, so a search test controls what is stored next to it. + if (ReplyToAddress != null) { - [NServiceBus.Headers.MessageId] = MessageId, - [NServiceBus.Headers.ProcessingEndpoint] = EndpointName, - [NServiceBus.Headers.ContentType] = ContentType, - [NServiceBus.Headers.EnclosedMessageTypes] = MessageType, - [NServiceBus.Headers.MessageIntent] = MessageIntent.ToString(), - ["NServiceBus.FailedQ"] = QueueAddress, - ["NServiceBus.ExceptionInfo.ExceptionType"] = ExceptionType, - ["NServiceBus.ExceptionInfo.Message"] = ExceptionMessage - }; + headers[NServiceBus.Headers.ReplyToAddress] = ReplyToAddress; + } + + headers[NServiceBus.Headers.ProcessingEndpoint] = EndpointName; + headers[NServiceBus.Headers.ContentType] = ContentType; + headers[NServiceBus.Headers.EnclosedMessageTypes] = MessageType; + headers[NServiceBus.Headers.MessageIntent] = MessageIntent.ToString(); + headers["NServiceBus.FailedQ"] = QueueAddress; + headers["NServiceBus.ExceptionInfo.ExceptionType"] = ExceptionType; + headers["NServiceBus.ExceptionInfo.Message"] = ExceptionMessage; if (ExceptionSource != null) { @@ -120,6 +125,7 @@ Dictionary BuildHeaders() MessageType = MessageType, ConversationId = ConversationId, QueueAddress = QueueAddress, + ReplyToAddress = ReplyToAddress, FailingEndpointAddress = FailingEndpointAddress, ExceptionType = ExceptionType, ExceptionMessage = ExceptionMessage,