diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index 9fe03e6c80a7..75f121db5ac7 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -81,7 +81,8 @@ - + + diff --git a/dotnet/src/Connectors/Connectors.Memory.Milvus/MilvusMemoryStore.cs b/dotnet/src/Connectors/Connectors.Memory.Milvus/MilvusMemoryStore.cs index c6d4f7a42b70..d69fa8bb5da4 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Milvus/MilvusMemoryStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Milvus/MilvusMemoryStore.cs @@ -21,6 +21,7 @@ public class MilvusMemoryStore : IMemoryStore, IDisposable { private readonly int _vectorSize; private readonly SimilarityMetricType _metricType; + private readonly ConsistencyLevel _consistencyLevel; private readonly bool _ownsMilvusClient; private readonly string _indexName; @@ -36,18 +37,10 @@ public class MilvusMemoryStore : IMemoryStore, IDisposable private const string TimestampFieldName = "timestamp"; private const int DefaultMilvusPort = 19530; - private const ConsistencyLevel DefaultConsistencyLevel = ConsistencyLevel.Session; private const int DefaultVarcharLength = 65_535; - private readonly QueryParameters _queryParametersWithEmbedding = new() - { - OutputFields = { IsReferenceFieldName, ExternalSourceNameFieldName, IdFieldName, DescriptionFieldName, TextFieldName, AdditionalMetadataFieldName, EmbeddingFieldName, KeyFieldName, TimestampFieldName } - }; - - private readonly QueryParameters _queryParametersWithoutEmbedding = new() - { - OutputFields = { IsReferenceFieldName, ExternalSourceNameFieldName, IdFieldName, DescriptionFieldName, TextFieldName, AdditionalMetadataFieldName, KeyFieldName, TimestampFieldName } - }; + private readonly QueryParameters _queryParametersWithEmbedding; + private readonly QueryParameters _queryParametersWithoutEmbedding; private readonly SearchParameters _searchParameters = new() { @@ -64,7 +57,7 @@ public class MilvusMemoryStore : IMemoryStore, IDisposable /// /// Creates a new , connecting to the given hostname on the default Milvus port of 19530. /// For more advanced configuration opens, construct a instance and pass it to - /// . + /// . /// /// The hostname or IP address to connect to. /// The port to connect to. Defaults to 19530. @@ -73,6 +66,7 @@ public class MilvusMemoryStore : IMemoryStore, IDisposable /// The name of the index to use. Defaults to . /// The size of the vectors used in Milvus. Defaults to 1536. /// The metric used to measure similarity between vectors. Defaults to . + /// The consistency level to be used in the search. Defaults to . /// An optional logger factory through which the Milvus client will log. public MilvusMemoryStore( string host, @@ -82,8 +76,11 @@ public MilvusMemoryStore( string? indexName = null, int vectorSize = 1536, SimilarityMetricType metricType = SimilarityMetricType.Ip, + ConsistencyLevel consistencyLevel = ConsistencyLevel.Session, ILoggerFactory? loggerFactory = null) - : this(new MilvusClient(host, port, ssl, database, callOptions: default, loggerFactory), indexName, vectorSize, metricType) + : this( + new MilvusClient(host, port, ssl, database, callOptions: default, loggerFactory), + indexName, vectorSize, metricType, consistencyLevel) { this._ownsMilvusClient = true; } @@ -91,7 +88,7 @@ public MilvusMemoryStore( /// /// Creates a new , connecting to the given hostname on the default Milvus port of 19530. /// For more advanced configuration opens, construct a instance and pass it to - /// . + /// . /// /// The hostname or IP address to connect to. /// The username to use for authentication. @@ -102,6 +99,7 @@ public MilvusMemoryStore( /// The name of the index to use. Defaults to . /// The size of the vectors used in Milvus. Defaults to 1536. /// The metric used to measure similarity between vectors. Defaults to . + /// The consistency level to be used in the search. Defaults to . /// An optional logger factory through which the Milvus client will log. public MilvusMemoryStore( string host, @@ -113,8 +111,11 @@ public MilvusMemoryStore( string? indexName = null, int vectorSize = 1536, SimilarityMetricType metricType = SimilarityMetricType.Ip, + ConsistencyLevel consistencyLevel = ConsistencyLevel.Session, ILoggerFactory? loggerFactory = null) - : this(new MilvusClient(host, username, password, port, ssl, database, callOptions: default, loggerFactory), indexName, vectorSize, metricType) + : this( + new MilvusClient(host, username, password, port, ssl, database, callOptions: default, loggerFactory), + indexName, vectorSize, metricType, consistencyLevel) { this._ownsMilvusClient = true; } @@ -122,7 +123,7 @@ public MilvusMemoryStore( /// /// Creates a new , connecting to the given hostname on the default Milvus port of 19530. /// For more advanced configuration opens, construct a instance and pass it to - /// . + /// . /// /// The hostname or IP address to connect to. /// An API key to be used for authentication, instead of a username and password. @@ -132,6 +133,7 @@ public MilvusMemoryStore( /// The name of the index to use. Defaults to . /// The size of the vectors used in Milvus. Defaults to 1536. /// The metric used to measure similarity between vectors. Defaults to . + /// The consistency level to be used in the search. Defaults to . /// An optional logger factory through which the Milvus client will log. public MilvusMemoryStore( string host, @@ -142,8 +144,11 @@ public MilvusMemoryStore( string? indexName = null, int vectorSize = 1536, SimilarityMetricType metricType = SimilarityMetricType.Ip, + ConsistencyLevel consistencyLevel = ConsistencyLevel.Session, ILoggerFactory? loggerFactory = null) - : this(new MilvusClient(host, apiKey, port, ssl, database, callOptions: default, loggerFactory), indexName, vectorSize, metricType) + : this( + new MilvusClient(host, apiKey, port, ssl, database, callOptions: default, loggerFactory), + indexName, vectorSize, metricType, consistencyLevel) { this._ownsMilvusClient = true; } @@ -155,27 +160,43 @@ public MilvusMemoryStore( /// The name of the index to use. Defaults to . /// The size of the vectors used in Milvus. Defaults to 1536. /// The metric used to measure similarity between vectors. Defaults to . + /// The consistency level to be used in the search. Defaults to . public MilvusMemoryStore( MilvusClient client, string? indexName = null, int vectorSize = 1536, - SimilarityMetricType metricType = SimilarityMetricType.Ip) - : this(client, ownsMilvusClient: false, indexName, vectorSize, metricType) + SimilarityMetricType metricType = SimilarityMetricType.Ip, + ConsistencyLevel consistencyLevel = ConsistencyLevel.Session) + : this(client, ownsMilvusClient: false, indexName, vectorSize, metricType, consistencyLevel) { } private MilvusMemoryStore( MilvusClient client, bool ownsMilvusClient, - string? indexName = null, - int vectorSize = 1536, - SimilarityMetricType metricType = SimilarityMetricType.Ip) + string? indexName, + int vectorSize, + SimilarityMetricType metricType, + ConsistencyLevel consistencyLevel) { this.Client = client; this._indexName = indexName ?? DefaultIndexName; this._vectorSize = vectorSize; this._metricType = metricType; this._ownsMilvusClient = ownsMilvusClient; + this._consistencyLevel = consistencyLevel; + + this._queryParametersWithEmbedding = new() + { + OutputFields = { IsReferenceFieldName, ExternalSourceNameFieldName, IdFieldName, DescriptionFieldName, TextFieldName, AdditionalMetadataFieldName, EmbeddingFieldName, KeyFieldName, TimestampFieldName }, + ConsistencyLevel = this._consistencyLevel + }; + + this._queryParametersWithoutEmbedding = new() + { + OutputFields = { IsReferenceFieldName, ExternalSourceNameFieldName, IdFieldName, DescriptionFieldName, TextFieldName, AdditionalMetadataFieldName, KeyFieldName, TimestampFieldName }, + ConsistencyLevel = this._consistencyLevel + }; } #endregion Constructors @@ -196,7 +217,7 @@ public async Task CreateCollectionAsync(string collectionName, CancellationToken EnableDynamicFields = true }; - MilvusCollection collection = await this.Client.CreateCollectionAsync(collectionName, schema, DefaultConsistencyLevel, cancellationToken: cancellationToken).ConfigureAwait(false); + MilvusCollection collection = await this.Client.CreateCollectionAsync(collectionName, schema, this._consistencyLevel, cancellationToken: cancellationToken).ConfigureAwait(false); await collection.CreateIndexAsync(EmbeddingFieldName, metricType: this._metricType, indexName: this._indexName, cancellationToken: cancellationToken).ConfigureAwait(false); await collection.WaitForIndexBuildAsync("float_vector", this._indexName, cancellationToken: cancellationToken).ConfigureAwait(false); @@ -228,8 +249,6 @@ public async Task UpsertAsync(string collectionName, MemoryRecord record { MilvusCollection collection = this.Client.GetCollection(collectionName); - await collection.DeleteAsync($@"{IdFieldName} in [""{record.Metadata.Id}""]", cancellationToken: cancellationToken).ConfigureAwait(false); - var metadata = record.Metadata; List fieldData = new() @@ -246,7 +265,7 @@ public async Task UpsertAsync(string collectionName, MemoryRecord record FieldData.Create(TimestampFieldName, new[] { record.Timestamp?.ToString(CultureInfo.InvariantCulture) ?? string.Empty }, isDynamic: true) }; - MutationResult result = await collection.InsertAsync(fieldData, cancellationToken: cancellationToken).ConfigureAwait(false); + MutationResult result = await collection.UpsertAsync(fieldData, cancellationToken: cancellationToken).ConfigureAwait(false); return result.Ids.StringIds![0]; } @@ -257,9 +276,6 @@ public async IAsyncEnumerable UpsertBatchAsync( IEnumerable records, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - // TODO: Milvus v2.3.0 will have a 1st-class upsert API which we should use. - // In the meantime, we do delete+insert, following the Python connector's example. - StringBuilder idString = new(); List isReferenceData = new(); @@ -295,7 +311,6 @@ public async IAsyncEnumerable UpsertBatchAsync( } MilvusCollection collection = this.Client.GetCollection(collectionName); - await collection.DeleteAsync($"{IdFieldName} in [{idString}]", cancellationToken: cancellationToken).ConfigureAwait(false); FieldData[] fieldData = { @@ -311,7 +326,7 @@ public async IAsyncEnumerable UpsertBatchAsync( FieldData.Create(TimestampFieldName, timestampData, isDynamic: true) }; - MutationResult result = await collection.InsertAsync(fieldData, cancellationToken: cancellationToken).ConfigureAwait(false); + MutationResult result = await collection.UpsertAsync(fieldData, cancellationToken: cancellationToken).ConfigureAwait(false); foreach (var id in result.Ids.StringIds!) { @@ -355,7 +370,10 @@ public async IAsyncEnumerable GetBatchAsync( IReadOnlyList fields = await this.Client .GetCollection(collectionName) - .QueryAsync($"{IdFieldName} in [{idString}]", withEmbeddings ? this._queryParametersWithEmbedding : this._queryParametersWithoutEmbedding, cancellationToken: cancellationToken) + .QueryAsync( + $"{IdFieldName} in [{idString}]", + withEmbeddings ? this._queryParametersWithEmbedding : this._queryParametersWithoutEmbedding, + cancellationToken: cancellationToken) .ConfigureAwait(false); var rowCount = fields[0].RowCount; diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Milvus/MilvusFixture.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Milvus/MilvusFixture.cs new file mode 100644 index 000000000000..876f8a3c5ad6 --- /dev/null +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Milvus/MilvusFixture.cs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Threading.Tasks; +using Milvus.Client; +using Testcontainers.Milvus; +using Xunit; + +namespace SemanticKernel.IntegrationTests.Connectors.Milvus; + +public sealed class MilvusFixture : IAsyncLifetime +{ + private readonly MilvusContainer _container = new MilvusBuilder().Build(); + + public string Host => this._container.Hostname; + public int Port => this._container.GetMappedPublicPort(MilvusBuilder.MilvusGrpcPort); + + public MilvusClient CreateClient() + => new(this.Host, "root", "milvus", this.Port); + + public Task InitializeAsync() + => this._container.StartAsync(); + + public Task DisposeAsync() + => this._container.DisposeAsync().AsTask(); +} diff --git a/dotnet/src/IntegrationTests/Connectors/Milvus/MilvusMemoryStoreTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Milvus/MilvusMemoryStoreTests.cs similarity index 93% rename from dotnet/src/IntegrationTests/Connectors/Milvus/MilvusMemoryStoreTests.cs rename to dotnet/src/IntegrationTests/Connectors/Memory/Milvus/MilvusMemoryStoreTests.cs index af3479fb8c9d..9f1b67ecdaf8 100644 --- a/dotnet/src/IntegrationTests/Connectors/Milvus/MilvusMemoryStoreTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Milvus/MilvusMemoryStoreTests.cs @@ -6,22 +6,19 @@ using System.Threading.Tasks; using Microsoft.SemanticKernel.Connectors.Milvus; using Microsoft.SemanticKernel.Memory; +using Milvus.Client; using Xunit; -namespace SemanticKernel.IntegrationTests.Milvus; +namespace SemanticKernel.IntegrationTests.Connectors.Milvus; -public class MilvusMemoryStoreTests : IAsyncLifetime +public class MilvusMemoryStoreTests : IClassFixture, IAsyncLifetime { - private const string MilvusHost = "127.0.0.1"; - private const int MilvusPort = 19530; - - // If null, all tests will be enabled - private const string SkipReason = "Requires Milvus up and running"; - private const string CollectionName = "test"; - private MilvusMemoryStore Store { get; set; } = new(MilvusHost, vectorSize: 5, port: MilvusPort); - [Fact(Skip = SkipReason)] + private readonly MilvusFixture _milvusFixture; + private MilvusMemoryStore Store { get; set; } = null!; + + [Fact] public async Task CreateCollectionAsync() { Assert.False(await this.Store.DoesCollectionExistAsync(CollectionName)); @@ -30,7 +27,7 @@ public async Task CreateCollectionAsync() Assert.True(await this.Store.DoesCollectionExistAsync(CollectionName)); } - [Fact(Skip = SkipReason)] + [Fact] public async Task DropCollectionAsync() { await this.Store.CreateCollectionAsync(CollectionName); @@ -38,7 +35,7 @@ public async Task DropCollectionAsync() Assert.False(await this.Store.DoesCollectionExistAsync(CollectionName)); } - [Fact(Skip = SkipReason)] + [Fact] public async Task GetCollectionsAsync() { await this.Store.CreateCollectionAsync("collection1"); @@ -49,7 +46,7 @@ public async Task GetCollectionsAsync() Assert.Contains("collection2", collections); } - [Fact(Skip = SkipReason)] + [Fact] public async Task UpsertAsync() { await this.Store.CreateCollectionAsync(CollectionName); @@ -69,7 +66,7 @@ public async Task UpsertAsync() Assert.Equal("Some id", id); } - [Theory(Skip = SkipReason)] + [Theory] [InlineData(true)] [InlineData(false)] public async Task GetAsync(bool withEmbeddings) @@ -94,7 +91,7 @@ public async Task GetAsync(bool withEmbeddings) record.Embedding.ToArray()); } - [Fact(Skip = SkipReason)] + [Fact] public async Task UpsertBatchAsync() { await this.Store.CreateCollectionAsync(CollectionName); @@ -105,7 +102,7 @@ public async Task UpsertBatchAsync() id => Assert.Equal("Some other id", id)); } - [Theory(Skip = SkipReason)] + [Theory] [InlineData(true)] [InlineData(false)] public async Task GetBatchAsync(bool withEmbeddings) @@ -148,18 +145,20 @@ public async Task GetBatchAsync(bool withEmbeddings) }); } - [Fact(Skip = SkipReason)] + [Fact] public async Task RemoveAsync() { await this.Store.CreateCollectionAsync(CollectionName); await this.InsertSampleDataAsync(); + using var milvusClient = this._milvusFixture.CreateClient(); + Assert.NotNull(await this.Store.GetAsync(CollectionName, "Some id")); await this.Store.RemoveAsync(CollectionName, "Some id"); Assert.Null(await this.Store.GetAsync(CollectionName, "Some id")); } - [Fact(Skip = SkipReason)] + [Fact] public async Task RemoveBatchAsync() { await this.Store.CreateCollectionAsync(CollectionName); @@ -172,7 +171,7 @@ public async Task RemoveBatchAsync() Assert.Null(await this.Store.GetAsync(CollectionName, "Some other id")); } - [Theory(Skip = SkipReason)] + [Theory] [InlineData(true)] [InlineData(false)] public async Task GetNearestMatchesAsync(bool withEmbeddings) @@ -221,7 +220,7 @@ public async Task GetNearestMatchesAsync(bool withEmbeddings) }); } - [Fact(Skip = SkipReason)] + [Fact] public async Task GetNearestMatchesWithMinRelevanceScoreAsync() { await this.Store.CreateCollectionAsync(CollectionName); @@ -238,7 +237,7 @@ public async Task GetNearestMatchesWithMinRelevanceScoreAsync() Assert.DoesNotContain(firstId, results.Select(r => r.Record.Metadata.Id)); } - [Theory(Skip = SkipReason)] + [Theory] [InlineData(true)] [InlineData(false)] public async Task GetNearestMatchAsync(bool withEmbeddings) @@ -297,8 +296,14 @@ private async Task> InsertSampleDataAsync() return idList; } + public MilvusMemoryStoreTests(MilvusFixture milvusFixture) + => this._milvusFixture = milvusFixture; + public async Task InitializeAsync() - => await this.Store.DeleteCollectionAsync(CollectionName); + { + this.Store = new(this._milvusFixture.Host, vectorSize: 5, port: this._milvusFixture.Port, consistencyLevel: ConsistencyLevel.Strong); + await this.Store.DeleteCollectionAsync(CollectionName); + } public Task DisposeAsync() { diff --git a/dotnet/src/IntegrationTests/IntegrationTests.csproj b/dotnet/src/IntegrationTests/IntegrationTests.csproj index cba7a606e9f1..033a8f4e30b7 100644 --- a/dotnet/src/IntegrationTests/IntegrationTests.csproj +++ b/dotnet/src/IntegrationTests/IntegrationTests.csproj @@ -46,6 +46,7 @@ all +