diff --git a/docs/testing.md b/docs/testing.md index af20a9d4f9..b5d731e7c5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -27,6 +27,8 @@ On Windows, [PackagedRavenDBTests](../src/ServiceControlInstaller.Engine.UnitTes Persistence tests check assumptions at the persistence seam level by exercising each persister. For local setup details, see [Local testing of persistence providers](testing-persistence.md). +The RavenDB persistence test projects contain an `IndexDefinitionApprovals` test that approves the definition of every index the persister deploys. RavenDB rebuilds an index side by side when its definition changes, and the rebuild of a large database can take days. When the test fails, record the changed indexes in the upgrade guide of the next release before you approve the new definition. An operator who locked an index must unlock it to receive the change. + ## Transport tests Transport tests run the transport test suite for each transport. diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/ApprovalFiles/IndexDefinitionApprovals.Verify_index_definitions_are_unchanged.approved.txt b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/ApprovalFiles/IndexDefinitionApprovals.Verify_index_definitions_are_unchanged.approved.txt new file mode 100644 index 0000000000..0ad2293026 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/ApprovalFiles/IndexDefinitionApprovals.Verify_index_definitions_are_unchanged.approved.txt @@ -0,0 +1,90 @@ +FailedAuditImportIndex + Maps: + docs.FailedAuditImports.Select(cc => new { + Id = Id(cc) + }) + +MessagesViewIndex + Maps: + docs.ProcessedMessages.Select(message => new { + MessageId = ((string) message.MessageMetadata["MessageId"]), + MessageType = ((string) message.MessageMetadata["MessageType"]), + IsSystemMessage = ((bool) message.MessageMetadata["IsSystemMessage"]), + Status = ((bool) message.MessageMetadata["IsRetried"]) ? "ResolvedSuccessfully" : "Successful", + TimeSent = ((DateTime) message.MessageMetadata["TimeSent"]), + ProcessedAt = message.ProcessedAt, + ReceivingEndpointName = ((message.MessageMetadata["ReceivingEndpoint"])).Name, + SendingEndpointName = ((message.MessageMetadata["SendingEndpoint"])).Name, + CriticalTime = ((TimeSpan ? ) message.MessageMetadata["CriticalTime"]), + ProcessingTime = ((TimeSpan ? ) message.MessageMetadata["ProcessingTime"]), + DeliveryTime = ((TimeSpan ? ) message.MessageMetadata["DeliveryTime"]), + Query = Enumerable.ToArray(DynamicEnumerable.Union(message.MessageMetadata.ToDictionary(e1 => e1.Key, e1 => e1.Value).Select(_ => _.Value.ToString()), new string[] { + String.Join(" ", message.Headers.ToDictionary(e1 => e1.Key, e1 => e1.Value).Select(x => x.Value)) + })), + ConversationId = ((string) message.MessageMetadata["ConversationId"]) + }) + Fields: + Query: Indexing=Search, Analyzer=Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net, Version=3.0.3.0, Culture=neutral, PublicKeyToken=85089178b9ac3181 + +MessagesViewIndexWithFullTextSearch + Maps: + docs.ProcessedMessages.Select(message => new { + MessageId = ((string) message.MessageMetadata["MessageId"]), + MessageType = ((string) message.MessageMetadata["MessageType"]), + IsSystemMessage = ((bool) message.MessageMetadata["IsSystemMessage"]), + Status = ((bool) message.MessageMetadata["IsRetried"]) ? "ResolvedSuccessfully" : "Successful", + TimeSent = ((DateTime) message.MessageMetadata["TimeSent"]), + ProcessedAt = message.ProcessedAt, + ReceivingEndpointName = ((message.MessageMetadata["ReceivingEndpoint"])).Name, + SendingEndpointName = ((message.MessageMetadata["SendingEndpoint"])).Name, + CriticalTime = ((TimeSpan ? ) message.MessageMetadata["CriticalTime"]), + ProcessingTime = ((TimeSpan ? ) message.MessageMetadata["ProcessingTime"]), + DeliveryTime = ((TimeSpan ? ) message.MessageMetadata["DeliveryTime"]), + Query = Enumerable.ToArray(DynamicEnumerable.Union(message.MessageMetadata.ToDictionary(e1 => e1.Key, e1 => e1.Value).Select(_ => _.Value.ToString()), new string[] { + String.Join(" ", message.Headers.ToDictionary(e1 => e1.Key, e1 => e1.Value).Select(x => x.Value)), + this.LoadAttachment(message, "body").GetContentAsString() + })), + ConversationId = ((string) message.MessageMetadata["ConversationId"]) + }) + Fields: + Query: Indexing=Search, Analyzer=Lucene.Net.Analysis.Standard.StandardAnalyzer, Lucene.Net, Version=3.0.3.0, Culture=neutral, PublicKeyToken=85089178b9ac3181 + +SagaDetailsIndex + Maps: + docs.SagaHistories.Select(doc => new { + SagaId = doc.SagaId, + Id = doc.SagaId, + SagaType = doc.SagaType, + Changes = doc.Changes + }) + docs.SagaSnapshots.Select(doc => new { + SagaId = doc.SagaId, + Id = doc.SagaId, + SagaType = doc.SagaType, + Changes = new[] { + new { + Endpoint = doc.Endpoint, + FinishTime = doc.FinishTime, + InitiatingMessage = doc.InitiatingMessage, + OutgoingMessages = doc.OutgoingMessages, + StartTime = doc.StartTime, + StateAfterChange = doc.StateAfterChange, + Status = doc.Status + } + } + }) + Reduce: + results.GroupBy(result => result.SagaId).Select(g => new { + g = g, + first = DynamicEnumerable.FirstOrDefault(g) + }).Select(this0 => new { + Id = this0.first.SagaId, + SagaId = this0.first.SagaId, + SagaType = this0.first.SagaType, + Changes = Enumerable.ToList(DynamicEnumerable.OrderByDescending(this0.g.SelectMany(x => x.Changes), x0 => x0.FinishTime).Take(50000)) + }) + Fields: + Changes: Indexing=No + SagaId: Indexing=Exact + SagaType: Indexing=No + diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexDefinitionApprovals.cs b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexDefinitionApprovals.cs new file mode 100644 index 0000000000..367ca98367 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexDefinitionApprovals.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Audit.Persistence.Tests; + +using NUnit.Framework; +using Particular.Approvals; +using Persistence.RavenDB; +using TestHelper; + +[TestFixture] +class IndexDefinitionApprovals +{ + // A changed definition makes RavenDB rebuild the index side by side at the next start-up, which can take days. + // Record the change in the upgrade guide before approving it, see docs/testing.md. + [Test] + public void Verify_index_definitions_are_unchanged() => + Approver.Verify(IndexDefinitionText.Render(typeof(DatabaseSetup).Assembly)); +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/ApprovalFiles/IndexDefinitionApprovals.Verify_index_definitions_are_unchanged.approved.txt b/src/ServiceControl.Persistence.Tests.RavenDB/ApprovalFiles/IndexDefinitionApprovals.Verify_index_definitions_are_unchanged.approved.txt new file mode 100644 index 0000000000..52f920f963 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/ApprovalFiles/IndexDefinitionApprovals.Verify_index_definitions_are_unchanged.approved.txt @@ -0,0 +1,223 @@ +ArchivedGroupsViewIndex + Maps: + docs.FailedMessages.Where(doc => doc.Status == "Archived").Select(doc => new { + doc = doc, + failureTimes = doc.ProcessingAttempts.Select(x => x.FailureDetails.TimeOfFailure) + }).SelectMany(this0 => this0.doc.FailureGroups, (this0, failureGroup) => new { + Id = failureGroup.Id, + Title = failureGroup.Title, + Count = 1, + First = DynamicEnumerable.Min(this0.failureTimes), + Last = DynamicEnumerable.Max(this0.failureTimes), + Type = failureGroup.Type + }) + Reduce: + results.GroupBy(result => new { + Id = result.Id, + Title = result.Title, + Type = result.Type + }).Select(g => new { + Id = g.Key.Id, + Title = g.Key.Title, + Count = Enumerable.Sum(g, x => ((int) x.Count)), + First = DynamicEnumerable.Min(g, x0 => ((DateTime) x0.First)), + Last = DynamicEnumerable.Max(g, x1 => ((DateTime) x1.Last)), + Type = g.Key.Type + }) + +CustomChecksIndex + Maps: + docs.CustomChecks.Select(cc => new { + Status = cc.Status, + ReportedAt = cc.ReportedAt, + Category = cc.Category, + CustomCheckId = cc.CustomCheckId + }) + +EndpointIndex + Maps: + docs.EndpointDocuments.Select(message => new { + ThroughputSource = message.EndpointId.ThroughputSource, + Name = message.EndpointId.Name + }) + +FailedErrorImportIndex + Maps: + docs.FailedErrorImports.Select(cc => new { + Id = Id(cc) + }) + +FailedMessageFacetsIndex + Maps: + docs.FailedMessages.Where(failure => failure.Status == "Unresolved").Select(failure => new { + failure = failure, + t = ((DynamicEnumerable.LastOrDefault(failure.ProcessingAttempts)).MessageMetadata["ReceivingEndpoint"]) + }).Select(this0 => new { + Name = this0.t.Name, + Host = this0.t.Host, + MessageType = (DynamicEnumerable.LastOrDefault(this0.failure.ProcessingAttempts)).MessageMetadata["MessageType"] + }) + Fields: + Host: Indexing=Exact + MessageType: Indexing=Exact + Name: Indexing=Exact + +FailedMessageRetries/ByBatch + Maps: + docs.FailedMessageRetries.Select(doc => new { + RetryBatchId = doc.RetryBatchId + }) + +FailedMessageViewIndex + Maps: + docs.FailedMessages.Select(message => new { + message = message, + processingAttemptsLast = DynamicEnumerable.LastOrDefault(message.ProcessingAttempts) + }).Select(this0 => new { + MessageId = this0.processingAttemptsLast.MessageMetadata["MessageId"], + MessageType = this0.processingAttemptsLast.MessageMetadata["MessageType"], + Status = this0.message.Status, + TimeSent = ((DateTime) this0.processingAttemptsLast.MessageMetadata["TimeSent"]), + ReceivingEndpointName = ((this0.processingAttemptsLast.MessageMetadata["ReceivingEndpoint"])).Name, + QueueAddress = this0.processingAttemptsLast.FailureDetails.AddressOfFailingEndpoint, + TimeOfFailure = this0.processingAttemptsLast.FailureDetails.TimeOfFailure, + LastModified = (this.MetadataFor(this0.message).Value < DateTime > ("@last-modified")).Ticks + }) + +FailedMessages/ByGroup + Maps: + docs.FailedMessages.Select(doc => new { + doc = doc, + processingAttemptsLast = DynamicEnumerable.LastOrDefault(doc.ProcessingAttempts) + }).SelectMany(this0 => this0.doc.FailureGroups, (this0, failureGroup) => new { + Id = Id(this0.doc), + MessageId = this0.doc.UniqueMessageId, + FailureGroupId = failureGroup.Id, + FailureGroupName = failureGroup.Title, + Status = this0.doc.Status, + MessageType = ((string) this0.processingAttemptsLast.MessageMetadata["MessageType"]), + TimeSent = ((DateTime) this0.processingAttemptsLast.MessageMetadata["TimeSent"]), + TimeOfFailure = this0.processingAttemptsLast.FailureDetails.TimeOfFailure, + LastModified = (this.MetadataFor(this0.doc).Value < DateTime > ("@last-modified")).Ticks + }) + Fields: + __all_fields: Storage=Yes + +FailureGroupsViewIndex + Maps: + docs.FailedMessages.Where(doc => doc.Status == "Unresolved").Select(doc => new { + doc = doc, + failureTimes = doc.ProcessingAttempts.Select(x => x.FailureDetails.TimeOfFailure) + }).SelectMany(this0 => this0.doc.FailureGroups, (this0, failureGroup) => new { + Id = failureGroup.Id, + Title = failureGroup.Title, + Count = 1, + First = DynamicEnumerable.Min(this0.failureTimes), + Last = DynamicEnumerable.Max(this0.failureTimes), + Type = failureGroup.Type + }) + Reduce: + results.GroupBy(result => new { + Id = result.Id, + Title = result.Title, + Type = result.Type + }).Select(g => new { + Id = g.Key.Id, + Title = g.Key.Title, + Count = Enumerable.Sum(g, x => ((int) x.Count)), + First = DynamicEnumerable.Min(g, x0 => ((DateTime) x0.First)), + Last = DynamicEnumerable.Max(g, x1 => ((DateTime) x1.Last)), + Type = g.Key.Type + }) + +GroupCommentIndex + Maps: + docs.GroupComments.Select(gc => new { + Id = Id(gc), + Comment = gc.Comment + }) + +KnownEndpointIndex + Maps: + docs.KnownEndpoints.Select(message => new { + EndpointDetails_Name = message.EndpointDetails.Name, + EndpointDetails_Host = message.EndpointDetails.Host, + HostDisplayName = message.HostDisplayName, + Monitored = message.Monitored + }) + +MessagesViewIndex + Maps: + docs.FailedMessages.Select(message => new { + message = message, + last = DynamicEnumerable.LastOrDefault(message.ProcessingAttempts) + }).Select(this0 => new { + MessageId = this0.last.MessageId, + MessageType = ((string) this0.last.MessageMetadata["MessageType"]), + IsSystemMessage = ((bool) this0.last.MessageMetadata["IsSystemMessage"]), + Status = this0.message.Status == "Archived" ? "ArchivedFailure" : (this0.message.Status == "Resolved" ? "ResolvedSuccessfully" : (this0.message.ProcessingAttempts.Count == 1 ? "Failed" : "RepeatedFailure")), + TimeSent = ((DateTime) this0.last.MessageMetadata["TimeSent"]), + ProcessedAt = this0.last.AttemptedAt, + ReceivingEndpointName = ((this0.last.MessageMetadata["ReceivingEndpoint"])).Name, + CriticalTime = ((TimeSpan ? ) this0.last.MessageMetadata["CriticalTime"]), + ProcessingTime = ((TimeSpan ? ) this0.last.MessageMetadata["ProcessingTime"]), + DeliveryTime = ((TimeSpan ? ) this0.last.MessageMetadata["DeliveryTime"]), + Query = Enumerable.ToArray(DynamicEnumerable.Union(this0.last.MessageMetadata.ToDictionary(e1 => e1.Key, e1 => e1.Value).Select(_ => _.Value.ToString()), new string[] { + String.Join(" ", this0.last.Headers.ToDictionary(e1 => e1.Key, e1 => e1.Value).Select(x => x.Value)) + })), + ConversationId = ((string) this0.last.MessageMetadata["ConversationId"]) + }) + Fields: + Query: Indexing=Search + +QueueAddressIndex + Maps: + docs.FailedMessages.Select(message => new { + message = message, + processingAttemptsLast = DynamicEnumerable.LastOrDefault(message.ProcessingAttempts) + }).Select(this0 => new { + PhysicalAddress = this0.processingAttemptsLast.FailureDetails.AddressOfFailingEndpoint, + FailedMessageCount = 1 + }) + Reduce: + results.GroupBy(result => result.PhysicalAddress).Select(g => new { + PhysicalAddress = g.Key, + FailedMessageCount = Enumerable.Sum(g, m => ((int) m.FailedMessageCount)) + }) + +RetryBatches/ByStatus/ReduceInitialBatchSize + Maps: + docs.RetryBatches.Select(doc => new { + RequestId = doc.RequestId, + RetryType = doc.RetryType, + HasStagingBatches = doc.Status == "Staging", + HasForwardingBatches = doc.Status == "Forwarding", + InitialBatchSize = doc.InitialBatchSize, + Originator = doc.Originator, + Classifier = doc.Classifier, + StartTime = doc.StartTime, + Last = doc.Last + }) + Reduce: + results.GroupBy(result => new { + RequestId = result.RequestId, + RetryType = result.RetryType + }).Select(g => new { + RequestId = g.Key.RequestId, + RetryType = g.Key.RetryType, + Originator = (DynamicEnumerable.FirstOrDefault(g)).Originator, + HasStagingBatches = Enumerable.Any(g, x => x.HasStagingBatches), + HasForwardingBatches = Enumerable.Any(g, x0 => x0.HasForwardingBatches), + InitialBatchSize = Enumerable.Sum(g, x1 => ((int) x1.InitialBatchSize)), + StartTime = (DynamicEnumerable.FirstOrDefault(g)).StartTime, + Last = DynamicEnumerable.Max(g, x2 => ((DateTime) x2.Last)), + Classifier = (DynamicEnumerable.FirstOrDefault(g)).Classifier + }) + +RetryBatches/ByStatusAndSession + Maps: + docs.RetryBatches.Select(doc => new { + RetrySessionId = doc.RetrySessionId, + Status = doc.Status + }) + diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/IndexDefinitionApprovals.cs b/src/ServiceControl.Persistence.Tests.RavenDB/IndexDefinitionApprovals.cs new file mode 100644 index 0000000000..7e9725b04b --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/IndexDefinitionApprovals.cs @@ -0,0 +1,17 @@ +namespace ServiceControl.Persistence.Tests.RavenDB +{ + using NUnit.Framework; + using Particular.Approvals; + using ServiceControl.Persistence.RavenDB; + using TestHelper; + + [TestFixture] + class IndexDefinitionApprovals + { + // A changed definition makes RavenDB rebuild the index side by side at the next start-up, which can take days. + // Record the change in the upgrade guide before approving it, see docs/testing.md. + [Test] + public void Verify_index_definitions_are_unchanged() => + Approver.Verify(IndexDefinitionText.Render(typeof(DatabaseSetup).Assembly)); + } +} diff --git a/src/TestHelper/IndexDefinitionText.cs b/src/TestHelper/IndexDefinitionText.cs new file mode 100644 index 0000000000..752b3a461e --- /dev/null +++ b/src/TestHelper/IndexDefinitionText.cs @@ -0,0 +1,139 @@ +namespace TestHelper; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; +using Raven.Client.Documents.Indexes; + +/// +/// Renders the parts of an index definition that RavenDB compares when ServiceControl deploys its indexes at +/// start-up. A difference in any of these parts makes RavenDB build a side-by-side replacement of the index. +/// +public static class IndexDefinitionText +{ + public static string Render(Assembly assembly) + { + // Same discovery as IndexDeployment.CreateIndexesAsync(Assembly, ...) + var indexes = assembly.GetTypes() + .Where(t => t.IsClass && !t.IsAbstract && t.IsSubclassOf(typeof(AbstractIndexCreationTask))) + .Select(t => (AbstractIndexCreationTask)Activator.CreateInstance(t)); + + return Render(indexes); + } + + public static string Render(IEnumerable indexes) + { + var builder = new StringBuilder(); + + foreach (var index in indexes.OrderBy(i => i.IndexName, StringComparer.Ordinal)) + { + Render(builder, index.CreateIndexDefinition()); + builder.AppendLine(); + } + + return builder.ToString(); + } + + static void Render(StringBuilder builder, IndexDefinition definition) + { + builder.AppendLine(definition.Name); + + AppendSection(builder, "Maps", definition.Maps.OrderBy(m => m, StringComparer.Ordinal), map => AppendBlock(builder, map)); + AppendSection(builder, "Reduce", [definition.Reduce], reduce => AppendBlock(builder, reduce)); + AppendSection(builder, "Fields", definition.Fields.OrderBy(f => f.Key, StringComparer.Ordinal), field => builder.AppendLine($" {field.Key}: {RenderFieldOptions(field.Value)}")); + AppendSection(builder, "Configuration", definition.Configuration.OrderBy(c => c.Key, StringComparer.Ordinal), setting => builder.AppendLine($" {setting.Key} = {setting.Value}")); + AppendSection(builder, "AdditionalSources", definition.AdditionalSources.OrderBy(s => s.Key, StringComparer.Ordinal), source => + { + builder.AppendLine($" {source.Key}:"); + AppendBlock(builder, source.Value, " "); + }); + AppendSection(builder, "AdditionalAssemblies", definition.AdditionalAssemblies.Select(a => a.ToString()).OrderBy(a => a, StringComparer.Ordinal), assembly => builder.AppendLine($" {assembly}")); + + AppendValue(builder, "OutputReduceToCollection", definition.OutputReduceToCollection); + AppendValue(builder, "PatternForOutputReduceToCollectionReferences", definition.PatternForOutputReduceToCollectionReferences); + AppendValue(builder, "PatternReferencesCollectionName", definition.PatternReferencesCollectionName); + AppendValue(builder, "LockMode", definition.LockMode); + AppendValue(builder, "Priority", definition.Priority); + AppendValue(builder, "State", definition.State); + AppendValue(builder, "DeploymentMode", definition.DeploymentMode); + } + + static void AppendSection(StringBuilder builder, string name, IEnumerable items, Action appendItem) + { + var list = items.Where(item => item is not string { Length: 0 } and not null).ToList(); + + if (list.Count == 0) + { + return; + } + + builder.AppendLine($" {name}:"); + + foreach (var item in list) + { + appendItem(item); + } + } + + static void AppendValue(StringBuilder builder, string name, object value) + { + if (value is null or string { Length: 0 }) + { + return; + } + + builder.AppendLine($" {name}: {value}"); + } + + static string RenderFieldOptions(IndexFieldOptions options) + { + var parts = new List(); + + if (options.Indexing.HasValue) + { + parts.Add($"Indexing={options.Indexing}"); + } + + if (options.Storage.HasValue) + { + parts.Add($"Storage={options.Storage}"); + } + + if (options.TermVector.HasValue) + { + parts.Add($"TermVector={options.TermVector}"); + } + + if (options.Analyzer != null) + { + parts.Add($"Analyzer={options.Analyzer}"); + } + + if (options.Suggestions.HasValue) + { + parts.Add($"Suggestions={options.Suggestions}"); + } + + if (options.Spatial != null) + { + parts.Add($"Spatial={options.Spatial.Type}/{options.Spatial.Strategy}/{options.Spatial.Units}/MaxTreeLevel={options.Spatial.MaxTreeLevel}"); + } + + return string.Join(", ", parts); + } + + static void AppendBlock(StringBuilder builder, string text, string indent = " ") + { + if (string.IsNullOrEmpty(text)) + { + return; + } + + foreach (var line in text.Split('\n')) + { + builder.Append(indent).AppendLine(line.TrimEnd('\r')); + } + } +}