.Net: Add Milvus vector store examples

dotnet/Directory.Packages.props

@@ -188,6 +188,7 @@
     <PackageVersion Include="CommunityToolkit.VectorData.AzureDocumentDB" Version="1.0.0" />
     <PackageVersion Include="CommunityToolkit.VectorData.AzureCosmosDB" Version="1.0.0" />
     <PackageVersion Include="CommunityToolkit.VectorData.InMemory" Version="1.0.1" />
+    <PackageVersion Include="CommunityToolkit.VectorData.Milvus" Version="1.0.0-preview" />
     <PackageVersion Include="CommunityToolkit.VectorData.PgVector" Version="1.0.0" />
     <PackageVersion Include="CommunityToolkit.VectorData.Qdrant" Version="1.0.0" />
     <PackageVersion Include="CommunityToolkit.VectorData.Redis" Version="1.0.0" />

dotnet/samples/Concepts/Concepts.csproj

@@ -54,6 +54,7 @@
     <PackageReference Include="CommunityToolkit.VectorData.AzureAISearch" />
     <PackageReference Include="CommunityToolkit.VectorData.AzureDocumentDB" />
     <PackageReference Include="CommunityToolkit.VectorData.InMemory" />
+    <PackageReference Include="CommunityToolkit.VectorData.Milvus" />
     <PackageReference Include="CommunityToolkit.VectorData.Qdrant" />
     <PackageReference Include="CommunityToolkit.VectorData.PgVector" />
     <PackageReference Include="CommunityToolkit.VectorData.Redis" />

dotnet/samples/Concepts/Memory/VectorStoreFixtures/VectorStoreInfra.cs

@@ -10,6 +10,44 @@ namespace Memory.VectorStoreFixtures;
 /// </summary>
 internal static class VectorStoreInfra
 {
+    /// <summary>
+    /// Set up a standalone Milvus container with embedded etcd and local storage.
+    /// </summary>
+    /// <param name="client">The docker client to create the container with.</param>
+    /// <returns>The id of the container.</returns>
+    public static async Task<string> SetupMilvusContainerAsync(DockerClient client)
+    {
+        await client.Images.CreateImageAsync(
+            new ImagesCreateParameters
+            {
+                FromImage = "milvusdb/milvus",
+                Tag = "v2.6.25",
+            },
+            null,
+            new Progress<JSONMessage>());
+
+        var container = await client.Containers.CreateContainerAsync(new CreateContainerParameters()
+        {
+            Image = "milvusdb/milvus:v2.6.25",
+            Cmd = ["milvus", "run", "standalone"],
+            Env = ["DEPLOY_MODE=STANDALONE", "ETCD_USE_EMBED=true", "COMMON_STORAGETYPE=local"],
+            HostConfig = new HostConfig()
+            {
+                PortBindings = new Dictionary<string, IList<PortBinding>>
+                {
+                    { "19530", new List<PortBinding> { new() { HostPort = "19530" } } },
+                },
+            },
+            ExposedPorts = new Dictionary<string, EmptyStruct>
+            {
+                { "19530", default },
+            },
+        });
+
+        await client.Containers.StartContainerAsync(container.ID, new ContainerStartParameters());
+        return container.ID;
+    }
+
     /// <summary>
     /// Setup the postgres pgvector container by pulling the image and running it.
     /// </summary>

dotnet/samples/Concepts/Memory/VectorStoreFixtures/VectorStoreMilvusContainerFixture.cs

@@ -0,0 +1,62 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Docker.DotNet;
+using Grpc.Core;
+using Milvus.Client;
+
+namespace Memory.VectorStoreFixtures;
+
+/// <summary>
+/// Fixture that starts a Milvus container for the samples and deletes it afterward.
+/// </summary>
+public class VectorStoreMilvusContainerFixture : IAsyncLifetime
+{
+    private DockerClient? _dockerClient;
+    private string? _milvusContainerId;
+
+    public Task InitializeAsync() => Task.CompletedTask;
+
+    public async Task ManualInitializeAsync()
+    {
+        if (this._milvusContainerId is null)
+        {
+            using var dockerClientConfiguration = new DockerClientConfiguration();
+            this._dockerClient = dockerClientConfiguration.CreateClient();
+            this._milvusContainerId = await VectorStoreInfra.SetupMilvusContainerAsync(this._dockerClient);
+        }
+
+        // Milvus must finish starting before the sample creates collections.
+        using var milvusClient = new MilvusClient("localhost");
+        using var timeout = new CancellationTokenSource(TimeSpan.FromMinutes(2));
+        while (true)
+        {
+            try
+            {
+                await milvusClient.ListCollectionsAsync(cancellationToken: timeout.Token);
+                return;
+            }
+            catch (RpcException) when (!timeout.IsCancellationRequested)
+            {
+                await Task.Delay(1000, timeout.Token);
+            }
+        }
+    }
+
+    public async Task DisposeAsync()
+    {
+        if (this._dockerClient is not null)
+        {
+            try
+            {
+                if (this._milvusContainerId is not null)
+                {
+                    await VectorStoreInfra.DeleteContainerAsync(this._dockerClient, this._milvusContainerId);
+                }
+            }
+            finally
+            {
+                this._dockerClient.Dispose();
+            }
+        }
+    }
+}

dotnet/samples/Concepts/Memory/VectorStore_VectorSearch_MultiStore_AzureAISearch.cs

@@ -22,6 +22,7 @@ namespace Memory;
 /// <para><see cref="VectorStore_VectorSearch_MultiStore_Redis"/></para>
 /// <para><see cref="VectorStore_VectorSearch_MultiStore_InMemory"/></para>
 /// <para><see cref="VectorStore_VectorSearch_MultiStore_Postgres"/></para>
+/// <para><see cref="VectorStore_VectorSearch_MultiStore_Milvus"/></para>
 ///
 /// To run this sample, you need an already existing Azure AI Search instance.
 /// To set your secrets use:

dotnet/samples/Concepts/Memory/VectorStore_VectorSearch_MultiStore_Common.cs

@@ -16,6 +16,7 @@ namespace Memory;
 /// <para><see cref="VectorStore_VectorSearch_MultiStore_Redis"/></para>
 /// <para><see cref="VectorStore_VectorSearch_MultiStore_InMemory"/></para>
 /// <para><see cref="VectorStore_VectorSearch_MultiStore_Postgres"/></para>
+/// <para><see cref="VectorStore_VectorSearch_MultiStore_Milvus"/></para>
 /// </summary>
 /// <param name="vectorStore">The vector store to ingest data into.</param>
 /// <param name="embeddingGenerator">The service to use for generating embeddings.</param>

dotnet/samples/Concepts/Memory/VectorStore_VectorSearch_MultiStore_InMemory.cs

@@ -20,6 +20,7 @@ namespace Memory;
 /// <para><see cref="VectorStore_VectorSearch_MultiStore_Redis"/></para>
 /// <para><see cref="VectorStore_VectorSearch_MultiStore_Qdrant"/></para>
 /// <para><see cref="VectorStore_VectorSearch_MultiStore_Postgres"/></para>
+/// <para><see cref="VectorStore_VectorSearch_MultiStore_Milvus"/></para>
 /// </summary>
 public class VectorStore_VectorSearch_MultiStore_InMemory(ITestOutputHelper output) : BaseTest(output)
 {

dotnet/samples/Concepts/Memory/VectorStore_VectorSearch_MultiStore_Milvus.cs

@@ -0,0 +1,86 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Azure.AI.OpenAI;
+using Azure.Identity;
+using CommunityToolkit.VectorData.Milvus;
+using Memory.VectorStoreFixtures;
+using Microsoft.Extensions.AI;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.SemanticKernel;
+using Milvus.Client;
+
+namespace Memory;
+
+/// <summary>
+/// An example showing how to use common code, that can work with any vector database, with a Milvus database.
+/// The common code is in the <see cref="VectorStore_VectorSearch_MultiStore_Common"/> class.
+/// The common code ingests data into the vector store and then searches over that data.
+/// This example is part of a set of examples each showing a different vector database.
+///
+/// For other databases, see the following classes:
+/// <para><see cref="VectorStore_VectorSearch_MultiStore_AzureAISearch"/></para>
+/// <para><see cref="VectorStore_VectorSearch_MultiStore_Qdrant"/></para>
+/// <para><see cref="VectorStore_VectorSearch_MultiStore_Redis"/></para>
+/// <para><see cref="VectorStore_VectorSearch_MultiStore_InMemory"/></para>
+/// <para><see cref="VectorStore_VectorSearch_MultiStore_Postgres"/></para>
+///
+/// To run this sample, you need a local instance of Docker running, since the associated fixture will try and start a Milvus container in the local docker instance.
+/// </summary>
+public class VectorStore_VectorSearch_MultiStore_Milvus(ITestOutputHelper output, VectorStoreMilvusContainerFixture milvusFixture) : BaseTest(output), IClassFixture<VectorStoreMilvusContainerFixture>
+{
+    [Fact]
+    public async Task ExampleWithDIAsync()
+    {
+        // Use the kernel for DI purposes.
+        var kernelBuilder = Kernel
+            .CreateBuilder();
+
+        // Register an embedding generation service with the DI container.
+        kernelBuilder.AddAzureOpenAIEmbeddingGenerator(
+            deploymentName: TestConfiguration.AzureOpenAIEmbeddings.DeploymentName,
+            endpoint: TestConfiguration.AzureOpenAIEmbeddings.Endpoint,
+            credential: new AzureCliCredential(),
+            dimensions: 1536);
+
+        // Initialize the Milvus docker container via the fixtures and register the Milvus VectorStore.
+        await milvusFixture.ManualInitializeAsync();
+        kernelBuilder.Services.AddMilvusVectorStore("localhost", https: false);
+
+        // Register the test output helper common processor with the DI container.
+        kernelBuilder.Services.AddSingleton<ITestOutputHelper>(this.Output);
+        kernelBuilder.Services.AddTransient<VectorStore_VectorSearch_MultiStore_Common>();
+
+        // Build the kernel.
+        var kernel = kernelBuilder.Build();
+
+        // Build a common processor object using the DI container.
+        var processor = kernel.GetRequiredService<VectorStore_VectorSearch_MultiStore_Common>();
+
+        // Run the process and pass a key generator function to it, to generate unique record keys.
+        // The key generator function is required, since different vector stores may require different key types.
+        // E.g. Milvus supports Guid, long and string keys, but others may support strings only.
+        await processor.IngestDataAndSearchAsync("skglossaryWithDI", () => Guid.NewGuid());
+    }
+
+    [Fact]
+    public async Task ExampleWithoutDIAsync()
+    {
+        // Create an embedding generation service.
+        var embeddingGenerator = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAIEmbeddings.Endpoint), new AzureCliCredential())
+            .GetEmbeddingClient(TestConfiguration.AzureOpenAIEmbeddings.DeploymentName)
+            .AsIEmbeddingGenerator(1536);
+
+        // Initialize the Milvus docker container via the fixtures and construct the Milvus VectorStore.
+        await milvusFixture.ManualInitializeAsync();
+        var milvusClient = new MilvusClient("localhost");
+        using var vectorStore = new MilvusVectorStore(milvusClient, ownsClient: true);
+
+        // Create the common processor that works for any vector store.
+        var processor = new VectorStore_VectorSearch_MultiStore_Common(vectorStore, embeddingGenerator, this.Output);
+
+        // Run the process and pass a key generator function to it, to generate unique record keys.
+        // The key generator function is required, since different vector stores may require different key types.
+        // E.g. Milvus supports Guid, long and string keys, but others may support strings only.
+        await processor.IngestDataAndSearchAsync("skglossaryWithoutDI", () => Guid.NewGuid());
+    }
+}

dotnet/samples/Concepts/Memory/VectorStore_VectorSearch_MultiStore_Postgres.cs

@@ -21,6 +21,7 @@ namespace Memory;
 /// <para><see cref="VectorStore_VectorSearch_MultiStore_Qdrant"/></para>
 /// <para><see cref="VectorStore_VectorSearch_MultiStore_Redis"/></para>
 /// <para><see cref="VectorStore_VectorSearch_MultiStore_InMemory"/></para>
+/// <para><see cref="VectorStore_VectorSearch_MultiStore_Milvus"/></para>
 ///
 /// To run this sample, you need a local instance of Docker running, since the associated fixture will try and start a Postgres container in the local docker instance.
 /// </summary>

dotnet/samples/Concepts/Memory/VectorStore_VectorSearch_MultiStore_Qdrant.cs

@@ -22,6 +22,7 @@ namespace Memory;
 /// <para><see cref="VectorStore_VectorSearch_MultiStore_Redis"/></para>
 /// <para><see cref="VectorStore_VectorSearch_MultiStore_InMemory"/></para>
 /// <para><see cref="VectorStore_VectorSearch_MultiStore_Postgres"/></para>
+/// <para><see cref="VectorStore_VectorSearch_MultiStore_Milvus"/></para>
 ///
 /// To run this sample, you need a local instance of Docker running, since the associated fixture will try and start a Qdrant container in the local docker instance.
 /// </summary>

dotnet/samples/Concepts/Memory/VectorStore_VectorSearch_MultiStore_Redis.cs

@@ -22,6 +22,7 @@ namespace Memory;
 /// <para><see cref="VectorStore_VectorSearch_MultiStore_Qdrant"/></para>
 /// <para><see cref="VectorStore_VectorSearch_MultiStore_InMemory"/></para>
 /// <para><see cref="VectorStore_VectorSearch_MultiStore_Postgres"/></para>
+/// <para><see cref="VectorStore_VectorSearch_MultiStore_Milvus"/></para>
 ///
 /// Redis supports two record storage types: Json and HashSet.
 /// Note the use of the <see cref="RedisStorageType"/> enum to specify the preferred storage type.

dotnet/samples/Demos/VectorStoreRAG/Options/ApplicationConfig.cs

@@ -17,6 +17,7 @@ internal sealed class ApplicationConfig
     private readonly AzureAISearchConfig _azureAISearchConfig = new();
     private readonly CosmosConfig _azureDocumentDBConfig = new();
     private readonly CosmosConfig _cosmosNoSqlConfig = new();
+    private readonly MilvusConfig _milvusConfig = new();
     private readonly QdrantConfig _qdrantConfig = new();
     private readonly RedisConfig _redisConfig = new();
     private readonly WeaviateConfig _weaviateConfig = new();
@@ -48,6 +49,9 @@ public ApplicationConfig(ConfigurationManager configurationManager)
         configurationManager
             .GetRequiredSection($"VectorStores:{CosmosConfig.NoSqlConfigSectionName}")
             .Bind(this._cosmosNoSqlConfig);
+        configurationManager
+            .GetRequiredSection($"VectorStores:{MilvusConfig.ConfigSectionName}")
+            .Bind(this._milvusConfig);
         configurationManager
             .GetRequiredSection($"VectorStores:{QdrantConfig.ConfigSectionName}")
             .Bind(this._qdrantConfig);
@@ -75,6 +79,8 @@ public ApplicationConfig(ConfigurationManager configurationManager)
 
     public CosmosConfig CosmosNoSqlConfig => this._cosmosNoSqlConfig;
 
+    public MilvusConfig MilvusConfig => this._milvusConfig;
+
     public QdrantConfig QdrantConfig => this._qdrantConfig;
 
     public RedisConfig RedisConfig => this._redisConfig;

dotnet/samples/Demos/VectorStoreRAG/Options/MilvusConfig.cs

@@ -0,0 +1,24 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.ComponentModel.DataAnnotations;
+
+namespace VectorStoreRAG.Options;
+
+/// <summary>
+/// Milvus service settings.
+/// </summary>
+internal sealed class MilvusConfig
+{
+    public const string ConfigSectionName = "Milvus";
+
+    [Required]
+    public string Host { get; set; } = string.Empty;
+
+    public int Port { get; set; } = 19530;
+
+    public bool Https { get; set; } = true;
+
+    public string? ApiKey { get; set; }
+
+    public string? Database { get; set; }
+}

dotnet/samples/Demos/VectorStoreRAG/Program.cs

@@ -92,6 +92,27 @@
         kernelBuilder.Services.AddInMemoryVectorStoreRecordCollection<string, TextSnippet<string>>(
             appConfig.RagConfig.CollectionName);
         break;
+    case "Milvus":
+        if (string.IsNullOrWhiteSpace(appConfig.MilvusConfig.ApiKey))
+        {
+            kernelBuilder.Services.AddMilvusCollection<string, TextSnippet<string>>(
+                appConfig.RagConfig.CollectionName,
+                appConfig.MilvusConfig.Host,
+                port: appConfig.MilvusConfig.Port,
+                https: appConfig.MilvusConfig.Https,
+                database: appConfig.MilvusConfig.Database);
+        }
+        else
+        {
+            kernelBuilder.Services.AddMilvusCollection<string, TextSnippet<string>>(
+                appConfig.RagConfig.CollectionName,
+                appConfig.MilvusConfig.Host,
+                apiKey: appConfig.MilvusConfig.ApiKey,
+                port: appConfig.MilvusConfig.Port,
+                https: appConfig.MilvusConfig.Https,
+                database: appConfig.MilvusConfig.Database);
+        }
+        break;
     case "Qdrant":
         kernelBuilder.Services.AddQdrantCollection<Guid, TextSnippet<Guid>>(
             appConfig.RagConfig.CollectionName,
@@ -123,6 +144,7 @@
     case "AzureDocumentDB":
     case "CosmosNoSql":
     case "InMemory":
+    case "Milvus":
     case "Redis":
         RegisterServices<string>(builder, kernelBuilder, appConfig);
         break;

dotnet/samples/Demos/VectorStoreRAG/README.md

@@ -12,6 +12,7 @@ The sample can be configured in various ways:
    1. AzureDocumentDB
    1. CosmosNoSql
    1. InMemory
+   1. Milvus
    1. Qdrant
    1. Redis
    1. Weaviate
@@ -130,6 +131,26 @@ dotnet user-secrets set "VectorStores:CosmosNoSql:ConnectionString" "<yourconnec
 dotnet user-secrets set "VectorStores:CosmosNoSql:DatabaseName" "<yourdbname>"
 ```
 
+### Milvus
+
+To use Milvus, set `Rag:VectorStoreType` to `Milvus`. The provider requires Milvus 2.6 or later.
+
+Start a local standalone instance with embedded etcd and local storage using Docker:
+
+```cli
+docker run -d --name milvus -p 19530:19530 -e DEPLOY_MODE=STANDALONE -e ETCD_USE_EMBED=true -e COMMON_STORAGETYPE=local milvusdb/milvus:v2.6.25 milvus run standalone
+```
+
+The default sample settings connect to this local instance without TLS or authentication. For a remote TLS endpoint, configure the host, port, database, and API key as needed:
+
+```cli
+dotnet user-secrets set "VectorStores:Milvus:Host" "<yourservice>"
+dotnet user-secrets set "VectorStores:Milvus:Port" "19530"
+dotnet user-secrets set "VectorStores:Milvus:Https" "true"
+dotnet user-secrets set "VectorStores:Milvus:Database" "default"
+dotnet user-secrets set "VectorStores:Milvus:ApiKey" "<yoursecret>"
+```
+
 ### Qdrant
 
 If you want to use Qdrant as your vector store, you will need to have an instance of Qdrant available.

dotnet/samples/Demos/VectorStoreRAG/VectorStoreRAG.csproj

@@ -21,6 +21,7 @@
     <PackageReference Include="CommunityToolkit.VectorData.AzureDocumentDB" />
     <PackageReference Include="CommunityToolkit.VectorData.AzureCosmosDB" />
     <PackageReference Include="CommunityToolkit.VectorData.InMemory" />
+    <PackageReference Include="CommunityToolkit.VectorData.Milvus" />
     <PackageReference Include="CommunityToolkit.VectorData.Qdrant" />
     <PackageReference Include="CommunityToolkit.VectorData.Redis" />
     <PackageReference Include="CommunityToolkit.VectorData.Weaviate" />