diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index b3941784aa08..e064461cfe16 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -65,7 +65,7 @@ - + diff --git a/dotnet/SK-dotnet.sln b/dotnet/SK-dotnet.sln index f9e83098f6ba..0a4bf8c39da2 100644 --- a/dotnet/SK-dotnet.sln +++ b/dotnet/SK-dotnet.sln @@ -487,6 +487,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ModelContextProtocol", "sam EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SqlServerIntegrationTests", "src\VectorDataIntegrationTests\SqlServerIntegrationTests\SqlServerIntegrationTests.csproj", "{A5E6193C-8431-4C6E-B674-682CB41EAA0C}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PineconeIntegrationTests", "src\VectorDataIntegrationTests\PineconeIntegrationTests\PineconeIntegrationTests.csproj", "{E9A74E0C-BC02-4DDD-A487-89847EDF8026}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1334,6 +1336,12 @@ Global {A5E6193C-8431-4C6E-B674-682CB41EAA0C}.Publish|Any CPU.Build.0 = Debug|Any CPU {A5E6193C-8431-4C6E-B674-682CB41EAA0C}.Release|Any CPU.ActiveCfg = Release|Any CPU {A5E6193C-8431-4C6E-B674-682CB41EAA0C}.Release|Any CPU.Build.0 = Release|Any CPU + {E9A74E0C-BC02-4DDD-A487-89847EDF8026}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E9A74E0C-BC02-4DDD-A487-89847EDF8026}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E9A74E0C-BC02-4DDD-A487-89847EDF8026}.Publish|Any CPU.ActiveCfg = Release|Any CPU + {E9A74E0C-BC02-4DDD-A487-89847EDF8026}.Publish|Any CPU.Build.0 = Release|Any CPU + {E9A74E0C-BC02-4DDD-A487-89847EDF8026}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E9A74E0C-BC02-4DDD-A487-89847EDF8026}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1516,6 +1524,7 @@ Global {8C658E1E-83C8-4127-B8BF-27A638A45DDD} = {6823CD5E-2ABE-41EB-B865-F86EC13F0CF9} {B16AC373-3DA8-4505-9510-110347CD635D} = {5D4C0700-BBB5-418F-A7B2-F392B9A18263} {A5E6193C-8431-4C6E-B674-682CB41EAA0C} = {4F381919-F1BE-47D8-8558-3187ED04A84F} + {E9A74E0C-BC02-4DDD-A487-89847EDF8026} = {4F381919-F1BE-47D8-8558-3187ED04A84F} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {FBDC56A3-86AD-4323-AA0F-201E59123B83} diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj b/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj index b2127f5131b0..3b51b03623ae 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj @@ -19,7 +19,7 @@ - + diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeFilterTranslator.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeFilterTranslator.cs new file mode 100644 index 000000000000..54a7202eaa07 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeFilterTranslator.cs @@ -0,0 +1,267 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Linq.Expressions; +using System.Reflection; +using System.Runtime.CompilerServices; +using Pinecone; + +namespace Microsoft.SemanticKernel.Connectors.Pinecone; + +// This class is a modification of MongoDBFilterTranslator that uses the same query language +// (https://docs.pinecone.io/guides/data/understanding-metadata#metadata-query-language), +// with the difference of representing everything as Metadata rather than BsonDocument. +// For representing collections of any kinds, we use List, +// as we sometimes need to extend the collection (with for example another condition). +internal class PineconeFilterTranslator +{ + private IReadOnlyDictionary _storagePropertyNames = null!; + private ParameterExpression _recordParameter = null!; + + internal Metadata Translate(LambdaExpression lambdaExpression, IReadOnlyDictionary storagePropertyNames) + { + this._storagePropertyNames = storagePropertyNames; + + Debug.Assert(lambdaExpression.Parameters.Count == 1); + this._recordParameter = lambdaExpression.Parameters[0]; + + return this.Translate(lambdaExpression.Body); + } + + private Metadata Translate(Expression? node) + => node switch + { + BinaryExpression + { + NodeType: ExpressionType.Equal or ExpressionType.NotEqual + or ExpressionType.GreaterThan or ExpressionType.GreaterThanOrEqual + or ExpressionType.LessThan or ExpressionType.LessThanOrEqual + } binary + => this.TranslateEqualityComparison(binary), + + BinaryExpression { NodeType: ExpressionType.AndAlso or ExpressionType.OrElse } andOr + => this.TranslateAndOr(andOr), + UnaryExpression { NodeType: ExpressionType.Not } not + => this.TranslateNot(not), + + // MemberExpression is generally handled within e.g. TranslateEqualityComparison; this is used to translate direct bool inside filter (e.g. Filter => r => r.Bool) + MemberExpression member when member.Type == typeof(bool) && this.TryTranslateFieldAccess(member, out _) + => this.TranslateEqualityComparison(Expression.Equal(member, Expression.Constant(true))), + + MethodCallExpression methodCall => this.TranslateMethodCall(methodCall), + + _ => throw new NotSupportedException("The following NodeType is unsupported: " + node?.NodeType) + }; + + private Metadata TranslateEqualityComparison(BinaryExpression binary) + { + if ((this.TryTranslateFieldAccess(binary.Left, out var storagePropertyName) && TryGetConstant(binary.Right, out var value)) + || (this.TryTranslateFieldAccess(binary.Right, out storagePropertyName) && TryGetConstant(binary.Left, out value))) + { + if (value is null) + { + throw new NotSupportedException("Pincone does not support null checks in vector search pre-filters"); + } + + // Short form of equality (instead of $eq) + if (binary.NodeType is ExpressionType.Equal) + { + return new Metadata { [storagePropertyName] = ToMetadata(value) }; + } + + var filterOperator = binary.NodeType switch + { + ExpressionType.NotEqual => "$ne", + ExpressionType.GreaterThan => "$gt", + ExpressionType.GreaterThanOrEqual => "$gte", + ExpressionType.LessThan => "$lt", + ExpressionType.LessThanOrEqual => "$lte", + + _ => throw new UnreachableException() + }; + + return new Metadata { [storagePropertyName] = new Metadata { [filterOperator] = ToMetadata(value) } }; + } + + throw new NotSupportedException("Invalid equality/comparison"); + } + + private Metadata TranslateAndOr(BinaryExpression andOr) + { + var mongoOperator = andOr.NodeType switch + { + ExpressionType.AndAlso => "$and", + ExpressionType.OrElse => "$or", + _ => throw new UnreachableException() + }; + + var (left, right) = (this.Translate(andOr.Left), this.Translate(andOr.Right)); + + List? nestedLeft = GetListOrNull(left, mongoOperator); + List? nestedRight = GetListOrNull(right, mongoOperator); + + switch ((nestedLeft, nestedRight)) + { + case (not null, not null): + nestedLeft.AddRange(nestedRight); + return left; + case (not null, null): + nestedLeft.Add(right); + return left; + case (null, not null): + nestedRight.Insert(0, left); + return right; + case (null, null): + return new Metadata { [mongoOperator] = new MetadataValue(new List { left, right }) }; + } + } + + private Metadata TranslateNot(UnaryExpression not) + { + switch (not.Operand) + { + // Special handling for !(a == b) and !(a != b) + case BinaryExpression { NodeType: ExpressionType.Equal or ExpressionType.NotEqual } binary: + return this.TranslateEqualityComparison( + Expression.MakeBinary( + binary.NodeType is ExpressionType.Equal ? ExpressionType.NotEqual : ExpressionType.Equal, + binary.Left, + binary.Right)); + + // Not over bool field (Filter => r => !r.Bool) + case MemberExpression member when member.Type == typeof(bool) && this.TryTranslateFieldAccess(member, out _): + return this.TranslateEqualityComparison(Expression.Equal(member, Expression.Constant(false))); + } + + var operand = this.Translate(not.Operand); + + // Identify NOT over $in, transform to $nin (https://www.mongodb.com/docs/manual/reference/operator/query/nin/#mongodb-query-op.-nin) + if (operand.Count == 1 && operand.First() is { Key: var fieldName, Value: MetadataValue nested } && nested.Value is Metadata nestedMetadata + && GetListOrNull(nestedMetadata, "$in") is List values) + { + return new Metadata { [fieldName] = new Metadata { ["$nin"] = values } }; + } + + throw new NotSupportedException("Pinecone does not support the NOT operator in vector search pre-filters"); + } + + private Metadata TranslateMethodCall(MethodCallExpression methodCall) + => methodCall switch + { + // Enumerable.Contains() + { Method.Name: nameof(Enumerable.Contains), Arguments: [var source, var item] } contains + when contains.Method.DeclaringType == typeof(Enumerable) + => this.TranslateContains(source, item), + + // List.Contains() + { + Method: + { + Name: nameof(Enumerable.Contains), + DeclaringType: { IsGenericType: true } declaringType + }, + Object: Expression source, + Arguments: [var item] + } when declaringType.GetGenericTypeDefinition() == typeof(List<>) => this.TranslateContains(source, item), + + _ => throw new NotSupportedException($"Unsupported method call: {methodCall.Method.DeclaringType?.Name}.{methodCall.Method.Name}") + }; + + private Metadata TranslateContains(Expression source, Expression item) + { + switch (source) + { + // Contains over array column (r => r.Strings.Contains("foo")) + case var _ when this.TryTranslateFieldAccess(source, out _): + throw new NotSupportedException("Pinecone does not support Contains within array fields ($elemMatch) in vector search pre-filters"); + + // Contains over inline enumerable + case NewArrayExpression newArray: + var elements = new object?[newArray.Expressions.Count]; + + for (var i = 0; i < newArray.Expressions.Count; i++) + { + if (!TryGetConstant(newArray.Expressions[i], out var elementValue)) + { + throw new NotSupportedException("Invalid element in array"); + } + + elements[i] = elementValue; + } + + return ProcessInlineEnumerable(elements, item); + + // Contains over captured enumerable (we inline) + case var _ when TryGetConstant(source, out var constantEnumerable) + && constantEnumerable is IEnumerable enumerable and not string: + return ProcessInlineEnumerable(enumerable, item); + + default: + throw new NotSupportedException("Unsupported Contains expression"); + } + + Metadata ProcessInlineEnumerable(IEnumerable elements, Expression item) + { + if (!this.TryTranslateFieldAccess(item, out var storagePropertyName)) + { + throw new NotSupportedException("Unsupported item type in Contains"); + } + + return new Metadata + { + [storagePropertyName] = new Metadata + { + ["$in"] = new MetadataValue(elements.Cast().Select(ToMetadata).ToList()) + } + }; + } + } + + private bool TryTranslateFieldAccess(Expression expression, [NotNullWhen(true)] out string? storagePropertyName) + { + if (expression is MemberExpression memberExpression && memberExpression.Expression == this._recordParameter) + { + if (!this._storagePropertyNames.TryGetValue(memberExpression.Member.Name, out storagePropertyName)) + { + throw new InvalidOperationException($"Property name '{memberExpression.Member.Name}' provided as part of the filter clause is not a valid property name."); + } + + return true; + } + + storagePropertyName = null; + return false; + } + + private static bool TryGetConstant(Expression expression, out object? constantValue) + { + switch (expression) + { + case ConstantExpression { Value: var v }: + constantValue = v; + return true; + + // This identifies compiler-generated closure types which contain captured variables. + case MemberExpression { Expression: ConstantExpression constant, Member: FieldInfo fieldInfo } + when constant.Type.Attributes.HasFlag(TypeAttributes.NestedPrivate) + && Attribute.IsDefined(constant.Type, typeof(CompilerGeneratedAttribute), inherit: true): + constantValue = fieldInfo.GetValue(constant.Value); + return true; + + default: + constantValue = null; + return false; + } + } + + private static MetadataValue? ToMetadata(object? value) + => value is null ? null : PineconeVectorStoreRecordFieldMapping.ConvertToMetadataValue(value); + + private static List? GetListOrNull(Metadata value, string mongoOperator) + => value.Count == 1 && value.First() is var element && element.Key == mongoOperator ? element.Value?.Value as List : null; +} diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeGenericDataModelMapper.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeGenericDataModelMapper.cs index 496a848ed394..df783a230498 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeGenericDataModelMapper.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeGenericDataModelMapper.cs @@ -34,7 +34,7 @@ public PineconeGenericDataModelMapper( /// public Vector MapFromDataToStorageModel(VectorStoreGenericDataModel dataModel) { - var metadata = new MetadataMap(); + var metadata = new Metadata(); // Map data properties. foreach (var dataProperty in this._propertyReader.DataProperties) @@ -42,9 +42,9 @@ public Vector MapFromDataToStorageModel(VectorStoreGenericDataModel data if (dataModel.Data.TryGetValue(dataProperty.DataModelPropertyName, out var propertyValue)) { var propertyStorageName = this._propertyReader.GetStoragePropertyName(dataProperty.DataModelPropertyName); - metadata[propertyStorageName] = propertyValue == null ? - new MetadataValue() : - PineconeVectorStoreRecordFieldMapping.ConvertToMetadataValue(propertyValue); + metadata[propertyStorageName] = propertyValue is not null + ? PineconeVectorStoreRecordFieldMapping.ConvertToMetadataValue(propertyValue) + : null; } } @@ -62,8 +62,8 @@ public Vector MapFromDataToStorageModel(VectorStoreGenericDataModel data // TODO: what about sparse values? var result = new Vector { - Id = (string)dataModel.Key, - Values = values.ToArray(), + Id = dataModel.Key, + Values = values, Metadata = metadata, SparseValues = null }; @@ -80,7 +80,7 @@ public VectorStoreGenericDataModel MapFromStorageToDataModel(Vector stor // Set Vector. if (options?.IncludeVectors is true) { - dataModel.Vectors.Add(this._propertyReader.FirstVectorPropertyName!, new ReadOnlyMemory(storageModel.Values)); + dataModel.Vectors.Add(this._propertyReader.FirstVectorPropertyName!, storageModel.Values); } // Set Data. @@ -91,9 +91,10 @@ public VectorStoreGenericDataModel MapFromStorageToDataModel(Vector stor var propertyStorageName = this._propertyReader.GetStoragePropertyName(dataProperty.DataModelPropertyName); if (storageModel.Metadata.TryGetValue(propertyStorageName, out var propertyValue)) { - dataModel.Data[dataProperty.DataModelPropertyName] = PineconeVectorStoreRecordFieldMapping.ConvertFromMetadataValueToNativeType( - propertyValue, - dataProperty.PropertyType); + dataModel.Data[dataProperty.DataModelPropertyName] = + propertyValue is not null + ? PineconeVectorStoreRecordFieldMapping.ConvertFromMetadataValueToNativeType(propertyValue, dataProperty.PropertyType) + : null; } } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs index 4f79810e641b..a072ea6e7336 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using System.Runtime.CompilerServices; using System.Threading; -using Grpc.Core; using Microsoft.Extensions.VectorData; using Pinecone; using Sdk = Pinecone; @@ -20,7 +19,6 @@ namespace Microsoft.SemanticKernel.Connectors.Pinecone; public class PineconeVectorStore : IVectorStore { private const string DatabaseName = "Pinecone"; - private const string ListCollectionsName = "ListCollections"; private readonly Sdk.PineconeClient _pineconeClient; private readonly PineconeVectorStoreOptions _options; @@ -63,24 +61,27 @@ public virtual IVectorStoreRecordCollection GetCollection public virtual async IAsyncEnumerable ListCollectionNamesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) { - IndexDetails[] collections; + IndexList indexList; try { - collections = await this._pineconeClient.ListIndexes(cancellationToken).ConfigureAwait(false); + indexList = await this._pineconeClient.ListIndexesAsync(cancellationToken: cancellationToken).ConfigureAwait(false); } - catch (RpcException ex) + catch (PineconeApiException ex) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { VectorStoreType = DatabaseName, - OperationName = ListCollectionsName + OperationName = "ListCollections" }; } - foreach (var collection in collections) + if (indexList.Indexes is not null) { - yield return collection.Name; + foreach (var index in indexList.Indexes) + { + yield return index.Name; + } } } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionCreateMapping.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionCreateMapping.cs deleted file mode 100644 index 5f8d6bf6137d..000000000000 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionCreateMapping.cs +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.Extensions.VectorData; -using Pinecone; - -namespace Microsoft.SemanticKernel.Connectors.Pinecone; - -/// -/// Contains mapping helpers to use when creating a Pinecone vector collection. -/// -internal static class PineconeVectorStoreCollectionCreateMapping -{ - /// - /// Maps information stored in to a structure used by Pinecone SDK to create a serverless index. - /// - /// The property to map. - /// The structure containing settings used to create a serverless index. - /// Thrown if the property is missing information or has unsupported options specified. - public static (uint Dimension, Metric Metric) MapServerlessIndex(VectorStoreRecordVectorProperty vectorProperty) - { - if (vectorProperty!.Dimensions is not > 0) - { - throw new InvalidOperationException($"Property {nameof(vectorProperty.Dimensions)} on {nameof(VectorStoreRecordVectorProperty)} '{vectorProperty.DataModelPropertyName}' must be set to a positive integer to create a collection."); - } - - return (Dimension: (uint)vectorProperty.Dimensions, Metric: GetSDKMetricAlgorithm(vectorProperty)); - } - - /// - /// Get the configured from the given . - /// If none is configured, the default is . - /// - /// The vector property definition. - /// The chosen . - /// Thrown if a distance function is chosen that isn't supported by Pinecone. - public static Metric GetSDKMetricAlgorithm(VectorStoreRecordVectorProperty vectorProperty) - => vectorProperty.DistanceFunction switch - { - DistanceFunction.CosineSimilarity => Metric.Cosine, - DistanceFunction.DotProductSimilarity => Metric.DotProduct, - DistanceFunction.EuclideanSquaredDistance => Metric.Euclidean, - null => Metric.Cosine, - _ => throw new InvalidOperationException($"Distance function '{vectorProperty.DistanceFunction}' for {nameof(VectorStoreRecordVectorProperty)} '{vectorProperty.DataModelPropertyName}' is not supported by the Pinecone VectorStore.") - }; -} diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionSearchMapping.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionSearchMapping.cs index 5b3d511c6b08..8e633c76e47e 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionSearchMapping.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionSearchMapping.cs @@ -14,15 +14,15 @@ internal static class PineconeVectorStoreCollectionSearchMapping { #pragma warning disable CS0618 // FilterClause is obsolete /// - /// Build a Pinecone from a set of filter clauses. + /// Build a Pinecone from a set of filter clauses. /// - /// The filter clauses to build the Pinecone from. + /// The filter clauses to build the Pinecone from. /// A mapping from property name to the name under which the property would be stored. - /// The Pinecone . + /// The Pinecone . /// Thrown for invalid property names, value types or filter clause types. - public static MetadataMap BuildSearchFilter(IEnumerable? filterClauses, IReadOnlyDictionary storagePropertyNamesMap) + public static Metadata BuildSearchFilter(IEnumerable? filterClauses, IReadOnlyDictionary storagePropertyNamesMap) { - var metadataMap = new MetadataMap(); + var metadataMap = new Metadata(); if (filterClauses is null) { @@ -46,15 +46,14 @@ public static MetadataMap BuildSearchFilter(IEnumerable? filterCla bool boolValue => (MetadataValue)boolValue, float floatValue => (MetadataValue)floatValue, double doubleValue => (MetadataValue)doubleValue, - decimal decimalValue => (MetadataValue)decimalValue, - _ => throw new InvalidOperationException($"Unsupported filter value type '{equalToFilterClause.Value.GetType().Name}'.") + _ => throw new NotSupportedException($"Unsupported filter value type '{equalToFilterClause.Value.GetType().Name}'.") }; metadataMap.Add(storagePropertyName, metadataValue); } else { - throw new InvalidOperationException($"Unsupported filter clause type '{filterClause.GetType().Name}'."); + throw new NotSupportedException($"Unsupported filter clause type '{filterClause.GetType().Name}'."); } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs index 6e44feda9334..3da753575141 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs @@ -3,13 +3,12 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Linq.Expressions; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; -using Grpc.Core; using Microsoft.Extensions.VectorData; using Pinecone; -using Pinecone.Grpc; using Sdk = Pinecone; namespace Microsoft.SemanticKernel.Connectors.Pinecone; @@ -23,14 +22,6 @@ public class PineconeVectorStoreRecordCollection : IVectorStoreRecordCo #pragma warning restore CA1711 // Identifiers should not have incorrect suffix { private const string DatabaseName = "Pinecone"; - private const string CreateCollectionName = "CreateCollection"; - private const string CollectionExistsName = "CollectionExists"; - private const string DeleteCollectionName = "DeleteCollection"; - - private const string UpsertOperationName = "Upsert"; - private const string DeleteOperationName = "Delete"; - private const string GetOperationName = "Get"; - private const string QueryOperationName = "Query"; private static readonly VectorSearchOptions s_defaultVectorSearchOptions = new(); @@ -38,8 +29,7 @@ public class PineconeVectorStoreRecordCollection : IVectorStoreRecordCo private readonly PineconeVectorStoreRecordCollectionOptions _options; private readonly VectorStoreRecordPropertyReader _propertyReader; private readonly IVectorStoreRecordMapper _mapper; - - private Sdk.Index? _index; + private IndexClient? _indexClient; /// public string CollectionName { get; } @@ -55,7 +45,8 @@ public class PineconeVectorStoreRecordCollection : IVectorStoreRecordCo public PineconeVectorStoreRecordCollection(Sdk.PineconeClient pineconeClient, string collectionName, PineconeVectorStoreRecordCollectionOptions? options = null) { Verify.NotNull(pineconeClient); - Verify.NotNullOrWhiteSpace(collectionName); + VerifyCollectionName(collectionName); + VectorStoreRecordPropertyVerification.VerifyGenericDataModelKeyType(typeof(TRecord), options?.VectorCustomMapper is not null, PineconeVectorStoreRecordFieldMapping.s_supportedKeyTypes); VectorStoreRecordPropertyVerification.VerifyGenericDataModelDefinitionSupplied(typeof(TRecord), options?.VectorStoreRecordDefinition is not null); @@ -90,36 +81,45 @@ public PineconeVectorStoreRecordCollection(Sdk.PineconeClient pineconeClient, st } /// - public virtual async Task CollectionExistsAsync(CancellationToken cancellationToken = default) - { - var result = await this.RunOperationAsync( - CollectionExistsName, + public virtual Task CollectionExistsAsync(CancellationToken cancellationToken = default) + => this.RunCollectionOperationAsync( + "CollectionExists", async () => { - var collections = await this._pineconeClient.ListIndexes(cancellationToken).ConfigureAwait(false); + var collections = await this._pineconeClient.ListIndexesAsync(cancellationToken: cancellationToken).ConfigureAwait(false); - return collections.Any(x => x.Name == this.CollectionName); - }).ConfigureAwait(false); - - return result; - } + return collections.Indexes?.Any(x => x.Name == this.CollectionName) is true; + }); /// - public virtual async Task CreateCollectionAsync(CancellationToken cancellationToken = default) + public virtual Task CreateCollectionAsync(CancellationToken cancellationToken = default) { // we already run through record property validation, so a single VectorStoreRecordVectorProperty is guaranteed. var vectorProperty = this._propertyReader.VectorProperty!; - var (dimension, metric) = PineconeVectorStoreCollectionCreateMapping.MapServerlessIndex(vectorProperty); - - await this.RunOperationAsync( - CreateCollectionName, - () => this._pineconeClient.CreateServerlessIndex( - this.CollectionName, - dimension, - metric, - this._options.ServerlessIndexCloud, - this._options.ServerlessIndexRegion, - cancellationToken)).ConfigureAwait(false); + + if (!string.IsNullOrEmpty(vectorProperty.IndexKind) && vectorProperty.IndexKind != "PGA") + { + throw new InvalidOperationException( + $"IndexKind of '{vectorProperty.IndexKind}' for property '{vectorProperty.DataModelPropertyName}' is not supported. Pinecone only supports 'PGA' (Pinecone Graph Algorithm), which is always enabled."); + } + + CreateIndexRequest request = new() + { + Name = this.CollectionName, + Dimension = vectorProperty.Dimensions ?? throw new InvalidOperationException($"Property {nameof(vectorProperty.Dimensions)} on {nameof(VectorStoreRecordVectorProperty)} '{vectorProperty.DataModelPropertyName}' must be set to a positive integer to create a collection."), + Metric = MapDistanceFunction(vectorProperty), + Spec = new ServerlessIndexSpec + { + Serverless = new ServerlessSpec + { + Cloud = MapCloud(this._options.ServerlessIndexCloud), + Region = this._options.ServerlessIndexRegion, + } + }, + }; + + return this.RunCollectionOperationAsync("CreateCollection", + () => this._pineconeClient.CreateIndexAsync(request, cancellationToken: cancellationToken)); } /// @@ -127,24 +127,66 @@ public virtual async Task CreateCollectionIfNotExistsAsync(CancellationToken can { if (!await this.CollectionExistsAsync(cancellationToken).ConfigureAwait(false)) { - await this.CreateCollectionAsync(cancellationToken).ConfigureAwait(false); + try + { + await this.CreateCollectionAsync(cancellationToken).ConfigureAwait(false); + } + catch (VectorStoreOperationException ex) when (ex.InnerException is PineconeApiException apiEx && apiEx.InnerException is ConflictError) + { + // If the collection got created in the meantime, we should ignore the exception. + } } } /// - public virtual Task DeleteCollectionAsync(CancellationToken cancellationToken = default) - => this.RunOperationAsync( - DeleteCollectionName, - () => this._pineconeClient.DeleteIndex(this.CollectionName, cancellationToken)); + public virtual async Task DeleteCollectionAsync(CancellationToken cancellationToken = default) + { + try + { + await this._pineconeClient.DeleteIndexAsync(this.CollectionName, cancellationToken: cancellationToken).ConfigureAwait(false); + } + catch (NotFoundError) + { + // If the collection does not exist, we should ignore the exception. + } + catch (PineconeApiException other) + { + throw new VectorStoreOperationException("Call to vector store failed.", other) + { + VectorStoreType = DatabaseName, + CollectionName = this.CollectionName, + OperationName = "DeleteCollection" + }; + } + } /// public virtual async Task GetAsync(string key, GetRecordOptions? options = null, CancellationToken cancellationToken = default) { Verify.NotNull(key); - var records = await this.GetBatchAsync([key], options, cancellationToken).ToListAsync(cancellationToken).ConfigureAwait(false); + Sdk.FetchRequest request = new() + { + Namespace = this._options.IndexNamespace, + Ids = [key] + }; - return records.FirstOrDefault(); + var response = await this.RunIndexOperationAsync( + "Get", + indexClient => indexClient.FetchAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false); + + var result = response.Vectors?.Values.FirstOrDefault(); + if (result is null) + { + return default; + } + + StorageToDataModelMapperOptions mapperOptions = new() { IncludeVectors = options?.IncludeVectors is true }; + return VectorStoreErrorHandler.RunModelConversion( + DatabaseName, + this.CollectionName, + "Get", + () => this._mapper.MapFromStorageToDataModel(result, mapperOptions)); } /// @@ -155,20 +197,32 @@ public virtual async IAsyncEnumerable GetBatchAsync( { Verify.NotNull(keys); - var indexNamespace = this.GetIndexNamespace(); - var mapperOptions = new StorageToDataModelMapperOptions { IncludeVectors = options?.IncludeVectors ?? false }; - - var index = await this.GetIndexAsync(this.CollectionName, cancellationToken).ConfigureAwait(false); + List keysList = keys.ToList(); + if (keysList.Count == 0) + { + yield break; + } - var results = await this.RunOperationAsync( - GetOperationName, - () => index.Fetch(keys, indexNamespace, cancellationToken)).ConfigureAwait(false); + Sdk.FetchRequest request = new() + { + Namespace = this._options.IndexNamespace, + Ids = keysList + }; + + var response = await this.RunIndexOperationAsync( + "GetBatch", + indexClient => indexClient.FetchAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false); + if (response.Vectors is null || response.Vectors.Count == 0) + { + yield break; + } + StorageToDataModelMapperOptions mapperOptions = new() { IncludeVectors = options?.IncludeVectors is true }; var records = VectorStoreErrorHandler.RunModelConversion( DatabaseName, this.CollectionName, - GetOperationName, - () => results.Values.Select(x => this._mapper.MapFromStorageToDataModel(x, mapperOptions))); + "GetBatch", + () => response.Vectors.Values.Select(x => this._mapper.MapFromStorageToDataModel(x, mapperOptions))); foreach (var record in records) { @@ -181,21 +235,37 @@ public virtual Task DeleteAsync(string key, CancellationToken cancellationToken { Verify.NotNullOrWhiteSpace(key); - return this.DeleteBatchAsync([key], cancellationToken); + Sdk.DeleteRequest request = new() + { + Namespace = this._options.IndexNamespace, + Ids = [key] + }; + + return this.RunIndexOperationAsync( + "Delete", + indexClient => indexClient.DeleteAsync(request, cancellationToken: cancellationToken)); } /// - public virtual async Task DeleteBatchAsync(IEnumerable keys, CancellationToken cancellationToken = default) + public virtual Task DeleteBatchAsync(IEnumerable keys, CancellationToken cancellationToken = default) { Verify.NotNull(keys); - var indexNamespace = this.GetIndexNamespace(); + List keysList = keys.ToList(); + if (keysList.Count == 0) + { + return Task.CompletedTask; + } - var index = await this.GetIndexAsync(this.CollectionName, cancellationToken).ConfigureAwait(false); + Sdk.DeleteRequest request = new() + { + Namespace = this._options.IndexNamespace, + Ids = keysList + }; - await this.RunOperationAsync( - DeleteOperationName, - () => index.Delete(keys, indexNamespace, cancellationToken)).ConfigureAwait(false); + return this.RunIndexOperationAsync( + "DeleteBatch", + indexClient => indexClient.DeleteAsync(request, cancellationToken: cancellationToken)); } /// @@ -203,19 +273,21 @@ public virtual async Task UpsertAsync(TRecord record, CancellationToken { Verify.NotNull(record); - var indexNamespace = this.GetIndexNamespace(); - - var index = await this.GetIndexAsync(this.CollectionName, cancellationToken).ConfigureAwait(false); - var vector = VectorStoreErrorHandler.RunModelConversion( DatabaseName, this.CollectionName, - UpsertOperationName, + "Upsert", () => this._mapper.MapFromDataToStorageModel(record)); - await this.RunOperationAsync( - UpsertOperationName, - () => index.Upsert([vector], indexNamespace, cancellationToken)).ConfigureAwait(false); + Sdk.UpsertRequest request = new() + { + Namespace = this._options.IndexNamespace, + Vectors = [vector], + }; + + await this.RunIndexOperationAsync( + "Upsert", + indexClient => indexClient.UpsertAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false); return vector.Id; } @@ -225,19 +297,26 @@ public virtual async IAsyncEnumerable UpsertBatchAsync(IEnumerable records.Select(this._mapper.MapFromDataToStorageModel).ToList()); - await this.RunOperationAsync( - UpsertOperationName, - () => index.Upsert(vectors, indexNamespace, cancellationToken)).ConfigureAwait(false); + if (vectors.Count == 0) + { + yield break; + } + + Sdk.UpsertRequest request = new() + { + Namespace = this._options.IndexNamespace, + Vectors = vectors, + }; + + await this.RunIndexOperationAsync( + "UpsertBatch", + indexClient => indexClient.UpsertAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false); foreach (var vector in vectors) { @@ -256,70 +335,73 @@ public virtual async Task> VectorizedSearchAsync).FullName}"); } - // Resolve options and build filter clause. - var internalOptions = options ?? s_defaultVectorSearchOptions; - var mapperOptions = new StorageToDataModelMapperOptions { IncludeVectors = options?.IncludeVectors ?? false }; + options ??= s_defaultVectorSearchOptions; -#pragma warning disable CS0618 // FilterClause is obsolete - var filter = PineconeVectorStoreCollectionSearchMapping.BuildSearchFilter( - internalOptions.OldFilter?.FilterClauses, - this._propertyReader.StoragePropertyNamesMap); +#pragma warning disable CS0618 // VectorSearchFilter is obsolete + var filter = options switch + { + { OldFilter: not null, Filter: not null } => throw new ArgumentException("Either Filter or OldFilter can be specified, but not both"), + { OldFilter: VectorSearchFilter legacyFilter } => PineconeVectorStoreCollectionSearchMapping.BuildSearchFilter(options.OldFilter?.FilterClauses, this._propertyReader.StoragePropertyNamesMap), + { Filter: Expression> newFilter } => new PineconeFilterTranslator().Translate(newFilter, this._propertyReader.StoragePropertyNamesMap), + _ => null + }; #pragma warning restore CS0618 - // Get the current index. - var indexNamespace = this.GetIndexNamespace(); - var index = await this.GetIndexAsync(this.CollectionName, cancellationToken).ConfigureAwait(false); - - // Search. - var results = await this.RunOperationAsync( - QueryOperationName, - () => index.Query( - floatVector.ToArray(), - (uint)(internalOptions.Skip + internalOptions.Top), - filter, - sparseValues: null, - indexNamespace, - internalOptions.IncludeVectors, - includeMetadata: true, - cancellationToken)).ConfigureAwait(false); - - // Skip the required results for paging. - var skippedResults = results.Skip(internalOptions.Skip); - - // Map the results. + Sdk.QueryRequest request = new() + { + TopK = (uint)(options.Top + options.Skip), + Namespace = this._options.IndexNamespace, + IncludeValues = options.IncludeVectors, + IncludeMetadata = true, + Vector = floatVector, + Filter = filter, + }; + + Sdk.QueryResponse response = await this.RunIndexOperationAsync( + "Query", + indexClient => indexClient.QueryAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false); + + if (response.Matches is null) + { + return new VectorSearchResults(Array.Empty>().ToAsyncEnumerable()); + } + + // Pinecone does not provide a way to skip results, so we need to do it manually. + var skippedResults = response.Matches + .Skip(options.Skip); + + StorageToDataModelMapperOptions mapperOptions = new() { IncludeVectors = options.IncludeVectors is true }; var records = VectorStoreErrorHandler.RunModelConversion( DatabaseName, this.CollectionName, - QueryOperationName, - () => + "Query", + () => skippedResults.Select(x => new VectorSearchResult(this._mapper.MapFromStorageToDataModel(new Sdk.Vector() { - // First convert to Vector objects, since the - // mapper requires these as input. - var vectorResults = skippedResults.Select(x => ( - Vector: new Vector() - { - Id = x.Id, - Values = x.Values ?? Array.Empty(), - Metadata = x.Metadata, - SparseValues = x.SparseValues - }, - x.Score)); - - return vectorResults.Select(x => new VectorSearchResult( - this._mapper.MapFromStorageToDataModel(x.Vector, mapperOptions), - x.Score)); - }); - - return new VectorSearchResults(records.ToAsyncEnumerable()); + Id = x.Id, + Values = x.Values ?? Array.Empty(), + Metadata = x.Metadata, + SparseValues = x.SparseValues + }, mapperOptions), x.Score))) + .ToAsyncEnumerable(); + + return new(records); } - private async Task RunOperationAsync(string operationName, Func> operation) + private async Task RunIndexOperationAsync(string operationName, Func> operation) { try { - return await operation.Invoke().ConfigureAwait(false); + if (this._indexClient is null) + { + // If we don't provide "host" to the Index method, it's going to perform + // a blocking call to DescribeIndexAsync!! + string hostName = (await this._pineconeClient.DescribeIndexAsync(this.CollectionName).ConfigureAwait(false)).Host; + this._indexClient = this._pineconeClient.Index(host: hostName); + } + + return await operation.Invoke(this._indexClient).ConfigureAwait(false); } - catch (RpcException ex) + catch (PineconeApiException ex) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { @@ -330,13 +412,13 @@ private async Task RunOperationAsync(string operationName, Func> o } } - private async Task RunOperationAsync(string operationName, Func operation) + private async Task RunCollectionOperationAsync(string operationName, Func> operation) { try { - await operation.Invoke().ConfigureAwait(false); + return await operation.Invoke().ConfigureAwait(false); } - catch (RpcException ex) + catch (PineconeApiException ex) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { @@ -347,13 +429,36 @@ private async Task RunOperationAsync(string operationName, Func operation) } } - private async Task> GetIndexAsync(string indexName, CancellationToken cancellationToken) + private static ServerlessSpecCloud MapCloud(string serverlessIndexCloud) + => serverlessIndexCloud switch + { + "aws" => ServerlessSpecCloud.Aws, + "azure" => ServerlessSpecCloud.Azure, + "gcp" => ServerlessSpecCloud.Gcp, + _ => throw new ArgumentException($"Invalid serverless index cloud: {serverlessIndexCloud}.", nameof(serverlessIndexCloud)) + }; + + private static CreateIndexRequestMetric MapDistanceFunction(VectorStoreRecordVectorProperty vectorProperty) + => vectorProperty.DistanceFunction switch + { + DistanceFunction.CosineSimilarity => CreateIndexRequestMetric.Cosine, + DistanceFunction.DotProductSimilarity => CreateIndexRequestMetric.Dotproduct, + DistanceFunction.EuclideanSquaredDistance => CreateIndexRequestMetric.Euclidean, + null => CreateIndexRequestMetric.Cosine, + _ => throw new NotSupportedException($"Distance function '{vectorProperty.DistanceFunction}' is not supported.") + }; + + private static void VerifyCollectionName(string collectionName) { - this._index ??= await this._pineconeClient.GetIndex(indexName, cancellationToken).ConfigureAwait(false); + Verify.NotNullOrWhiteSpace(collectionName); - return this._index; + // Based on https://docs.pinecone.io/troubleshooting/restrictions-on-index-names + foreach (char character in collectionName) + { + if (!((character is >= 'a' and <= 'z') || character is '-' || (character is >= '0' and <= '9'))) + { + throw new ArgumentException("Collection name must contain only ASCII lowercase letters, digits and dashes.", nameof(collectionName)); + } + } } - - private string? GetIndexNamespace() - => this._options.IndexNamespace; } diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordFieldMapping.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordFieldMapping.cs index 6acbec24d72c..9573740f8580 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordFieldMapping.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordFieldMapping.cs @@ -48,7 +48,7 @@ internal static class PineconeVectorStoreRecordFieldMapping ]; public static object? ConvertFromMetadataValueToNativeType(MetadataValue metadataValue, Type targetType) - => metadataValue.Inner switch + => metadataValue.Value switch { null => null, bool boolValue => boolValue, @@ -59,26 +59,28 @@ internal static class PineconeVectorStoreRecordFieldMapping long longValue => ConvertToNumericValue(longValue, targetType), float floatValue => ConvertToNumericValue(floatValue, targetType), double doubleValue => ConvertToNumericValue(doubleValue, targetType), - decimal decimalValue => ConvertToNumericValue(decimalValue, targetType), MetadataValue[] array => VectorStoreRecordMapping.CreateEnumerable(array.Select(x => ConvertFromMetadataValueToNativeType(x, VectorStoreRecordPropertyVerification.GetCollectionElementType(targetType))), targetType), List list => VectorStoreRecordMapping.CreateEnumerable(list.Select(x => ConvertFromMetadataValueToNativeType(x, VectorStoreRecordPropertyVerification.GetCollectionElementType(targetType))), targetType), - _ => throw new VectorStoreRecordMappingException($"Unsupported metadata type: '{metadataValue.Inner?.GetType().FullName}'."), + _ => throw new VectorStoreRecordMappingException($"Unsupported metadata type: '{metadataValue.Value?.GetType().FullName}'."), }; - // TODO: take advantage of MetadataValue.TryCreate once we upgrade the version of Pinecone.NET public static MetadataValue ConvertToMetadataValue(object? sourceValue) => sourceValue switch { bool boolValue => boolValue, + bool[] bools => bools, + List bools => bools, string stringValue => stringValue, + string[] stringArray => stringArray, + List stringList => stringList, + double doubleValue => doubleValue, + double[] doubles => doubles, + List doubles => doubles, + // Other numeric types are simply cast into double in implicit way. + // We could consider supporting arrays of these types. int intValue => intValue, long longValue => longValue, float floatValue => floatValue, - double doubleValue => doubleValue, - decimal decimalValue => decimalValue, - string[] stringArray => stringArray, - List stringList => stringList, - IEnumerable stringEnumerable => stringEnumerable.ToArray(), _ => throw new VectorStoreRecordMappingException($"Unsupported source value type '{sourceValue?.GetType().FullName}'.") }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordMapper.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordMapper.cs index 501937eaf50d..1163c1a66bea 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordMapper.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordMapper.cs @@ -40,7 +40,7 @@ public Vector MapFromDataToStorageModel(TRecord dataModel) throw new VectorStoreRecordMappingException($"Key property {this._propertyReader.KeyPropertyName} on provided record of type {typeof(TRecord).FullName} may not be null."); } - var metadata = new MetadataMap(); + var metadata = new Metadata(); foreach (var dataPropertyInfo in this._propertyReader.DataPropertiesInfo) { var propertyName = this._propertyReader.GetStoragePropertyName(dataPropertyInfo.Name); @@ -61,7 +61,7 @@ public Vector MapFromDataToStorageModel(TRecord dataModel) var result = new Vector { Id = (string)keyObject, - Values = values.ToArray(), + Values = values, Metadata = metadata, SparseValues = null }; @@ -83,7 +83,7 @@ public TRecord MapFromStorageToDataModel(Vector storageModel, StorageToDataModel { this._propertyReader.FirstVectorPropertyInfo!.SetValue( outputRecord, - new ReadOnlyMemory(storageModel.Values)); + storageModel.Values); } // Set Data. @@ -94,7 +94,7 @@ public TRecord MapFromStorageToDataModel(Vector storageModel, StorageToDataModel this._propertyReader.DataPropertiesInfo, this._propertyReader.StoragePropertyNamesMap, storageModel.Metadata, - PineconeVectorStoreRecordFieldMapping.ConvertFromMetadataValueToNativeType); + PineconeVectorStoreRecordFieldMapping.ConvertFromMetadataValueToNativeType!); } return outputRecord; diff --git a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeGenericDataModelMapperTests.cs b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeGenericDataModelMapperTests.cs index 9a41450d9649..0a96bb41cc3b 100644 --- a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeGenericDataModelMapperTests.cs +++ b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeGenericDataModelMapperTests.cs @@ -77,20 +77,20 @@ public void MapFromDataToStorageModelMapsAllSupportedTypes() // Assert Assert.Equal(TestKeyString, storageModel.Id); - Assert.Equal("string", (string?)storageModel.Metadata!["StringDataProp"].Inner); + Assert.Equal("string", (string?)storageModel.Metadata!["StringDataProp"]!.Value); // MetadataValue converts all numeric types to double. - Assert.Equal(1, (double?)storageModel.Metadata["IntDataProp"].Inner); - Assert.Equal(2, (double?)storageModel.Metadata["NullableIntDataProp"].Inner); - Assert.Equal(3L, (double?)storageModel.Metadata["LongDataProp"].Inner); - Assert.Equal(4L, (double?)storageModel.Metadata["NullableLongDataProp"].Inner); - Assert.Equal(5.0f, (double?)storageModel.Metadata["FloatDataProp"].Inner); - Assert.Equal(6.0f, (double?)storageModel.Metadata["NullableFloatDataProp"].Inner); - Assert.Equal(7.0, (double?)storageModel.Metadata["DoubleDataProp"].Inner); - Assert.Equal(8.0, (double?)storageModel.Metadata["NullableDoubleDataProp"].Inner); - Assert.Equal(true, (bool?)storageModel.Metadata["BoolDataProp"].Inner); - Assert.Equal(false, (bool?)storageModel.Metadata["NullableBoolDataProp"].Inner); - Assert.Equal(s_taglist, ((IEnumerable?)(storageModel.Metadata["TagListDataProp"].Inner!)) - .Select(x => x.Inner as string) + Assert.Equal(1, (double?)storageModel.Metadata["IntDataProp"]!.Value); + Assert.Equal(2, (double?)storageModel.Metadata["NullableIntDataProp"]!.Value); + Assert.Equal(3L, (double?)storageModel.Metadata["LongDataProp"]!.Value); + Assert.Equal(4L, (double?)storageModel.Metadata["NullableLongDataProp"]!.Value); + Assert.Equal(5.0f, (double?)storageModel.Metadata["FloatDataProp"]!.Value); + Assert.Equal(6.0f, (double?)storageModel.Metadata["NullableFloatDataProp"]!.Value); + Assert.Equal(7.0, (double?)storageModel.Metadata["DoubleDataProp"]!.Value); + Assert.Equal(8.0, (double?)storageModel.Metadata["NullableDoubleDataProp"]!.Value); + Assert.Equal(true, (bool?)storageModel.Metadata["BoolDataProp"]!.Value); + Assert.Equal(false, (bool?)storageModel.Metadata["NullableBoolDataProp"]!.Value); + Assert.Equal(s_taglist, ((IEnumerable?)(storageModel.Metadata["TagListDataProp"]!.Value!)) + .Select(x => x.Value as string) .ToArray()); Assert.Equal(s_vector, storageModel.Values); } @@ -136,9 +136,9 @@ public void MapFromDataToStorageModelMapsNullValues() // Assert Assert.Equal(TestKeyString, storageModel.Id); - Assert.True(storageModel.Metadata!["StringDataProp"].Inner == null); - Assert.True(storageModel.Metadata["NullableIntDataProp"].Inner == null); - Assert.True(storageModel.Metadata["NullableTagListDataProp"].Inner == null); + Assert.Null(storageModel.Metadata!["StringDataProp"]); + Assert.Null(storageModel.Metadata["NullableIntDataProp"]); + Assert.Null(storageModel.Metadata["NullableTagListDataProp"]); } [Fact] @@ -153,7 +153,7 @@ public void MapFromStorageToDataModelMapsAllSupportedTypes() var storageModel = new Vector() { Id = TestKeyString, - Metadata = new MetadataMap() + Metadata = new Metadata() { ["StringDataProp"] = (MetadataValue)"string", ["IntDataProp"] = (MetadataValue)1, @@ -168,7 +168,7 @@ public void MapFromStorageToDataModelMapsAllSupportedTypes() ["NullableBoolDataProp"] = (MetadataValue)false, ["TagListDataProp"] = (MetadataValue)new MetadataValue[] { "tag1", "tag2" } }, - Values = [1.0f, 2.0f, 3.0f] + Values = new float[] { 1.0f, 2.0f, 3.0f } }; // Act @@ -210,13 +210,13 @@ public void MapFromStorageToDataModelMapsNullValues() var storageModel = new Vector() { Id = TestKeyString, - Metadata = new MetadataMap() + Metadata = new Metadata() { - ["StringDataProp"] = new MetadataValue(), - ["NullableIntDataProp"] = new MetadataValue(), - ["NullableTagListDataProp"] = new MetadataValue(), + ["StringDataProp"] = null, + ["NullableIntDataProp"] = null, + ["NullableTagListDataProp"] = null, }, - Values = [1.0f, 2.0f, 3.0f] + Values = new float[] { 1.0f, 2.0f, 3.0f } }; var reader = new VectorStoreRecordPropertyReader( @@ -327,7 +327,7 @@ public void MapFromStorageToDataModelSkipsMissingProperties() var storageModel = new Vector() { Id = TestKeyString, - Values = [1.0f, 2.0f, 3.0f] + Values = new float[] { 1.0f, 2.0f, 3.0f } }; // Act diff --git a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeKernelBuilderExtensionsTests.cs b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeKernelBuilderExtensionsTests.cs index 6777470e537d..21f7b6649da5 100644 --- a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeKernelBuilderExtensionsTests.cs +++ b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeKernelBuilderExtensionsTests.cs @@ -26,7 +26,7 @@ public PineconeKernelBuilderExtensionsTests() public void AddVectorStoreRegistersClass() { // Arrange. - using var client = new Sdk.PineconeClient("fake api key"); + var client = new Sdk.PineconeClient("fake api key"); this._kernelBuilder.Services.AddSingleton(client); // Act. @@ -50,7 +50,7 @@ public void AddVectorStoreWithApiKeyRegistersClass() public void AddVectorStoreRecordCollectionRegistersClass() { // Arrange. - using var client = new Sdk.PineconeClient("fake api key"); + var client = new Sdk.PineconeClient("fake api key"); this._kernelBuilder.Services.AddSingleton(client); // Act. diff --git a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeServiceCollectionExtensionsTests.cs b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeServiceCollectionExtensionsTests.cs index 084222d37bda..736cc3e3839d 100644 --- a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeServiceCollectionExtensionsTests.cs +++ b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeServiceCollectionExtensionsTests.cs @@ -26,7 +26,7 @@ public PineconeServiceCollectionExtensionsTests() public void AddVectorStoreRegistersClass() { // Arrange. - using var client = new Sdk.PineconeClient("fake api key"); + var client = new Sdk.PineconeClient("fake api key"); this._serviceCollection.AddSingleton(client); // Act. @@ -49,7 +49,7 @@ public void AddVectorStoreWithApiKeyRegistersClass() public void AddVectorStoreRecordCollectionRegistersClass() { // Arrange. - using var client = new Sdk.PineconeClient("fake api key"); + var client = new Sdk.PineconeClient("fake api key"); this._serviceCollection.AddSingleton(client); // Act. diff --git a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeVectorStoreRecordCollectionTests.cs b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeVectorStoreRecordCollectionTests.cs index 85ed14f7a468..0dc2620140f3 100644 --- a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeVectorStoreRecordCollectionTests.cs +++ b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeVectorStoreRecordCollectionTests.cs @@ -35,7 +35,7 @@ public void CanCreateCollectionWithMismatchedDefinitionAndType() new VectorStoreRecordVectorProperty("Embedding", typeof(ReadOnlyMemory)) { Dimensions = 4 }, } }; - using var pineconeClient = new Sdk.PineconeClient("fake api key"); + var pineconeClient = new Sdk.PineconeClient("fake api key"); // Act. var sut = new PineconeVectorStoreRecordCollection( diff --git a/dotnet/src/Experimental/Process.IntegrationTestRunner.Dapr/Process.IntegrationTestRunner.Dapr.csproj b/dotnet/src/Experimental/Process.IntegrationTestRunner.Dapr/Process.IntegrationTestRunner.Dapr.csproj index ba07a9a9dcad..2d35183b3648 100644 --- a/dotnet/src/Experimental/Process.IntegrationTestRunner.Dapr/Process.IntegrationTestRunner.Dapr.csproj +++ b/dotnet/src/Experimental/Process.IntegrationTestRunner.Dapr/Process.IntegrationTestRunner.Dapr.csproj @@ -32,7 +32,6 @@ - diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/CommonPineconeVectorStoreRecordCollectionTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/CommonPineconeVectorStoreRecordCollectionTests.cs deleted file mode 100644 index 21964a61e3d0..000000000000 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/CommonPineconeVectorStoreRecordCollectionTests.cs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using Microsoft.Extensions.VectorData; -using Microsoft.SemanticKernel.Connectors.Pinecone; -using SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone.Xunit; -using Xunit; - -namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone; - -/// -/// Inherits common integration tests that should pass for any . -/// -/// Pinecone setup and teardown. -[Collection("PineconeVectorStoreTests")] -[PineconeApiKeySetCondition] -public class CommonPineconeVectorStoreRecordCollectionTests(PineconeVectorStoreFixture fixture) : BaseVectorStoreRecordCollectionTests, IClassFixture -{ - protected override string Key1 => "1"; - protected override string Key2 => "2"; - protected override string Key3 => "3"; - protected override string Key4 => "4"; - - protected override int DelayAfterIndexCreateInMilliseconds => 2000; - - protected override int DelayAfterUploadInMilliseconds => 15000; - - [SuppressMessage("Globalization", "CA1308:Normalize strings to uppercase", Justification = "Pinecone collection names should be lower case.")] - protected override IVectorStoreRecordCollection GetTargetRecordCollection(string recordCollectionName, VectorStoreRecordDefinition? vectorStoreRecordDefinition) - { - return new PineconeVectorStoreRecordCollection(fixture.Client, recordCollectionName.ToLowerInvariant(), new() - { - VectorStoreRecordDefinition = vectorStoreRecordDefinition - }); - } - - protected override HashSet GetSupportedDistanceFunctions() - { - return [DistanceFunction.CosineSimilarity, DistanceFunction.DotProductSimilarity, DistanceFunction.EuclideanSquaredDistance]; - } -} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeAllTypes.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeAllTypes.cs deleted file mode 100644 index 7067781987bc..000000000000 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeAllTypes.cs +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using Microsoft.Extensions.VectorData; - -namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone; - -#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. -public record PineconeAllTypes() -{ - [VectorStoreRecordKey] - public string Id { get; init; } - - [VectorStoreRecordData] - public bool BoolProperty { get; set; } - [VectorStoreRecordData] - public bool? NullableBoolProperty { get; set; } - [VectorStoreRecordData] - public string StringProperty { get; set; } - [VectorStoreRecordData] - public string? NullableStringProperty { get; set; } - [VectorStoreRecordData] - public int IntProperty { get; set; } - [VectorStoreRecordData] - public int? NullableIntProperty { get; set; } - [VectorStoreRecordData] - public long LongProperty { get; set; } - [VectorStoreRecordData] - public long? NullableLongProperty { get; set; } - [VectorStoreRecordData] - public float FloatProperty { get; set; } - [VectorStoreRecordData] - public float? NullableFloatProperty { get; set; } - [VectorStoreRecordData] - public double DoubleProperty { get; set; } - [VectorStoreRecordData] - public double? NullableDoubleProperty { get; set; } - [VectorStoreRecordData] - public decimal DecimalProperty { get; set; } - [VectorStoreRecordData] - public decimal? NullableDecimalProperty { get; set; } - -#pragma warning disable CA1819 // Properties should not return arrays - [VectorStoreRecordData] - public string[] StringArray { get; set; } - [VectorStoreRecordData] - public string[]? NullableStringArray { get; set; } -#pragma warning restore CA1819 // Properties should not return arrays - - [VectorStoreRecordData] - public List StringList { get; set; } - [VectorStoreRecordData] - public List? NullableStringList { get; set; } - - [VectorStoreRecordData] - public IReadOnlyCollection Collection { get; set; } - [VectorStoreRecordData] - public IEnumerable Enumerable { get; set; } - - [VectorStoreRecordVector(Dimensions: 8, DistanceFunction: DistanceFunction.DotProductSimilarity)] - public ReadOnlyMemory? Embedding { get; set; } -} -#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeHotel.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeHotel.cs deleted file mode 100644 index 54185830d5c0..000000000000 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeHotel.cs +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Text.Json.Serialization; -using Microsoft.Extensions.VectorData; - -namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone; - -#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. -public record PineconeHotel() - -{ - [VectorStoreRecordKey] - public string HotelId { get; init; } - - [VectorStoreRecordData] - public string HotelName { get; set; } - - [JsonPropertyName("code_of_the_hotel")] - [VectorStoreRecordData] - public int HotelCode { get; set; } - - [VectorStoreRecordData] - public float HotelRating { get; set; } - - [JsonPropertyName("json_parking")] - [VectorStoreRecordData(StoragePropertyName = "parking_is_included")] - public bool ParkingIncluded { get; set; } - - [VectorStoreRecordData] - public List Tags { get; set; } = []; - - [VectorStoreRecordData] - public string Description { get; set; } - - [VectorStoreRecordVector(Dimensions: 8, DistanceFunction: DistanceFunction.DotProductSimilarity)] - public ReadOnlyMemory DescriptionEmbedding { get; set; } -} -#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeUserSecretsExtensions.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeUserSecretsExtensions.cs deleted file mode 100644 index 1644b7427e99..000000000000 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeUserSecretsExtensions.cs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.IO; -using System.Reflection; -using System.Text.Json; -using Microsoft.Extensions.Configuration.UserSecrets; - -namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone; -public static class PineconeUserSecretsExtensions -{ - public const string PineconeApiKeyUserSecretEntry = "PineconeApiKey"; - - public static string ReadPineconeApiKey() - => JsonSerializer.Deserialize>( - File.ReadAllText(PathHelper.GetSecretsPathFromSecretsId( - typeof(PineconeUserSecretsExtensions).Assembly.GetCustomAttribute()! - .UserSecretsId)))![PineconeApiKeyUserSecretEntry].Trim(); - - public static bool ContainsPineconeApiKey() - { - var userSecretsIdAttribute = typeof(PineconeUserSecretsExtensions).Assembly.GetCustomAttribute(); - if (userSecretsIdAttribute == null) - { - return false; - } - - var path = PathHelper.GetSecretsPathFromSecretsId(userSecretsIdAttribute.UserSecretsId); - if (!File.Exists(path)) - { - return false; - } - - return JsonSerializer.Deserialize>( - File.ReadAllText(path))!.ContainsKey(PineconeApiKeyUserSecretEntry); - } -} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreFixture.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreFixture.cs deleted file mode 100644 index c0c002f22ba0..000000000000 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreFixture.cs +++ /dev/null @@ -1,350 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net.Http; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using Microsoft.Extensions.VectorData; -using Microsoft.SemanticKernel.Connectors.Pinecone; -using Pinecone.Grpc; -using Xunit; -using Sdk = Pinecone; - -namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone; - -public class PineconeVectorStoreFixture : IAsyncLifetime -{ - private const int MaxAttemptCount = 100; - private const int DelayInterval = 300; - - public string IndexName { get; } = "sk-index" -#pragma warning disable CA1308 // Normalize strings to uppercase - + new Regex("[^a-zA-Z0-9]", RegexOptions.None, matchTimeout: new TimeSpan(0, 0, 10)).Replace(Environment.MachineName.ToLowerInvariant(), ""); -#pragma warning restore CA1308 // Normalize strings to uppercase - - public Sdk.PineconeClient Client { get; private set; } = null!; - public PineconeVectorStore VectorStore { get; private set; } = null!; - public PineconeVectorStoreRecordCollection HotelRecordCollection { get; set; } = null!; - public PineconeVectorStoreRecordCollection AllTypesRecordCollection { get; set; } = null!; - public PineconeVectorStoreRecordCollection HotelRecordCollectionWithCustomNamespace { get; set; } = null!; - public IVectorStoreRecordCollection HotelRecordCollectionFromVectorStore { get; set; } = null!; - public IVectorStoreRecordCollection> HotelRecordCollectionWithGenericDataModel { get; set; } = null!; - - public virtual Sdk.Index Index { get; set; } = null!; - - public virtual async Task InitializeAsync() - { - this.Client = new Sdk.PineconeClient(PineconeUserSecretsExtensions.ReadPineconeApiKey()); - this.VectorStore = new PineconeVectorStore(this.Client); - - var hotelRecordDefinition = new VectorStoreRecordDefinition - { - Properties = - [ - new VectorStoreRecordKeyProperty(nameof(PineconeHotel.HotelId), typeof(string)), - new VectorStoreRecordDataProperty(nameof(PineconeHotel.HotelName), typeof(string)), - new VectorStoreRecordDataProperty(nameof(PineconeHotel.HotelCode), typeof(int)), - new VectorStoreRecordDataProperty(nameof(PineconeHotel.ParkingIncluded), typeof(bool)) { StoragePropertyName = "parking_is_included" }, - new VectorStoreRecordDataProperty(nameof(PineconeHotel.HotelRating), typeof(float)), - new VectorStoreRecordDataProperty(nameof(PineconeHotel.Tags), typeof(List)), - new VectorStoreRecordDataProperty(nameof(PineconeHotel.Description), typeof(string)), - new VectorStoreRecordVectorProperty(nameof(PineconeHotel.DescriptionEmbedding), typeof(ReadOnlyMemory)) { Dimensions = 8, DistanceFunction = DistanceFunction.DotProductSimilarity } - ] - }; - - var allTypesRecordDefinition = new VectorStoreRecordDefinition - { - Properties = - [ - new VectorStoreRecordKeyProperty(nameof(PineconeAllTypes.Id), typeof(string)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.BoolProperty), typeof(bool)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableBoolProperty), typeof(bool?)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.StringProperty), typeof(string)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableStringProperty), typeof(string)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.IntProperty), typeof(int)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableIntProperty), typeof(int?)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.LongProperty), typeof(long)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableLongProperty), typeof(long?)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.FloatProperty), typeof(float)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableFloatProperty), typeof(float?)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.DoubleProperty), typeof(double)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableDoubleProperty), typeof(double?)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.DecimalProperty), typeof(decimal)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableDecimalProperty), typeof(decimal?)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.StringArray), typeof(string[])), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableStringArray), typeof(string[])), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.StringList), typeof(List)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableStringList), typeof(List)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.Collection), typeof(IReadOnlyCollection)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.Enumerable), typeof(IEnumerable)), - new VectorStoreRecordVectorProperty(nameof(PineconeAllTypes.Embedding), typeof(ReadOnlyMemory?)) { Dimensions = 8, DistanceFunction = DistanceFunction.DotProductSimilarity } - ] - }; - - this.HotelRecordCollection = new PineconeVectorStoreRecordCollection( - this.Client, - this.IndexName, - new PineconeVectorStoreRecordCollectionOptions - { - VectorStoreRecordDefinition = hotelRecordDefinition - }); - - this.AllTypesRecordCollection = new PineconeVectorStoreRecordCollection( - this.Client, - this.IndexName, - new PineconeVectorStoreRecordCollectionOptions - { - VectorStoreRecordDefinition = allTypesRecordDefinition - }); - - this.HotelRecordCollectionWithCustomNamespace = new PineconeVectorStoreRecordCollection( - this.Client, - this.IndexName, - new PineconeVectorStoreRecordCollectionOptions - { - VectorStoreRecordDefinition = hotelRecordDefinition, - IndexNamespace = "my-namespace" - }); - - this.HotelRecordCollectionFromVectorStore = this.VectorStore.GetCollection( - this.IndexName, - hotelRecordDefinition); - - this.HotelRecordCollectionWithGenericDataModel = this.VectorStore.GetCollection>( - this.IndexName, - hotelRecordDefinition); - - await this.ClearIndexesAsync(); - await this.CreateIndexAndWaitAsync(); - await this.AddSampleDataAsync(); - } - - private async Task CreateIndexAndWaitAsync() - { - var attemptCount = 0; - - await this.HotelRecordCollection.CreateCollectionAsync(); - - do - { - await Task.Delay(DelayInterval); - attemptCount++; - this.Index = await this.Client.GetIndex(this.IndexName); - } while (!this.Index.Status.IsReady && attemptCount <= MaxAttemptCount); - - if (!this.Index.Status.IsReady) - { - throw new InvalidOperationException("'Create index' operation didn't complete in time. Index name: " + this.IndexName); - } - } - - public async Task DisposeAsync() - { - if (this.Client is not null) - { - await this.ClearIndexesAsync(); - this.Client.Dispose(); - } - } - - private async Task AddSampleDataAsync() - { - var fiveSeasons = new PineconeHotel - { - HotelId = "five-seasons", - HotelName = "Five Seasons Hotel", - Description = "Great service any season.", - HotelCode = 7, - HotelRating = 4.5f, - ParkingIncluded = true, - DescriptionEmbedding = new ReadOnlyMemory([7.5f, 71.0f, 71.5f, 72.0f, 72.5f, 73.0f, 73.5f, 74.0f]), - Tags = ["wi-fi", "sauna", "gym", "pool"] - }; - - var vacationInn = new PineconeHotel - { - HotelId = "vacation-inn", - HotelName = "Vacation Inn Hotel", - Description = "On vacation? Stay with us.", - HotelCode = 11, - HotelRating = 4.3f, - ParkingIncluded = true, - DescriptionEmbedding = new ReadOnlyMemory([17.5f, 721.0f, 731.5f, 742.0f, 762.5f, 783.0f, 793.5f, 704.0f]), - Tags = ["wi-fi", "breakfast", "gym"] - }; - - var bestEastern = new PineconeHotel - { - HotelId = "best-eastern", - HotelName = "Best Eastern Hotel", - Description = "Best hotel east of New York.", - HotelCode = 42, - HotelRating = 4.7f, - ParkingIncluded = true, - DescriptionEmbedding = new ReadOnlyMemory([47.5f, 421.0f, 741.5f, 744.0f, 742.5f, 483.0f, 743.5f, 744.0f]), - Tags = ["wi-fi", "breakfast", "gym"] - }; - - var stats = await this.Index.DescribeStats(); - var vectorCountBefore = stats.TotalVectorCount; - - // use both Upsert and BatchUpsert methods and also use record collections created directly and using vector store - await this.HotelRecordCollection.UpsertAsync(fiveSeasons); - vectorCountBefore = await this.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: 1); - - await this.HotelRecordCollectionFromVectorStore.UpsertBatchAsync([vacationInn, bestEastern]).ToListAsync(); - vectorCountBefore = await this.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: 2); - - var allTypes1 = new PineconeAllTypes - { - Id = "all-types-1", - BoolProperty = true, - NullableBoolProperty = false, - StringProperty = "string prop 1", - NullableStringProperty = "nullable prop 1", - IntProperty = 1, - NullableIntProperty = 10, - LongProperty = 100L, - NullableLongProperty = 1000L, - FloatProperty = 10.5f, - NullableFloatProperty = 100.5f, - DoubleProperty = 23.75d, - NullableDoubleProperty = 233.75d, - DecimalProperty = 50.75m, - NullableDecimalProperty = 500.75m, - StringArray = ["one", "two"], - NullableStringArray = ["five", "six"], - StringList = ["eleven", "twelve"], - NullableStringList = ["fifteen", "sixteen"], - Collection = ["Foo", "Bar"], - Enumerable = ["another", "and another"], - Embedding = new ReadOnlyMemory([1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f, 8.5f]) - }; - - var allTypes2 = new PineconeAllTypes - { - Id = "all-types-2", - BoolProperty = false, - NullableBoolProperty = null, - StringProperty = "string prop 2", - NullableStringProperty = null, - IntProperty = 2, - NullableIntProperty = null, - LongProperty = 200L, - NullableLongProperty = null, - FloatProperty = 20.5f, - NullableFloatProperty = null, - DoubleProperty = 43.75, - NullableDoubleProperty = null, - DecimalProperty = 250.75M, - NullableDecimalProperty = null, - StringArray = [], - NullableStringArray = null, - StringList = [], - NullableStringList = null, - Collection = [], - Enumerable = [], - Embedding = new ReadOnlyMemory([10.5f, 20.5f, 30.5f, 40.5f, 50.5f, 60.5f, 70.5f, 80.5f]) - }; - - await this.AllTypesRecordCollection.UpsertBatchAsync([allTypes1, allTypes2]).ToListAsync(); - vectorCountBefore = await this.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: 2); - - var custom = new PineconeHotel - { - HotelId = "custom-hotel", - HotelName = "Custom Hotel", - Description = "Everything customizable!", - HotelCode = 17, - HotelRating = 4.25f, - ParkingIncluded = true, - DescriptionEmbedding = new ReadOnlyMemory([147.5f, 1421.0f, 1741.5f, 1744.0f, 1742.5f, 1483.0f, 1743.5f, 1744.0f]), - }; - - await this.HotelRecordCollectionWithCustomNamespace.UpsertAsync(custom); - vectorCountBefore = await this.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: 1); - } - - public async Task VerifyVectorCountModifiedAsync(uint vectorCountBefore, int delta) - { - var attemptCount = 0; - Sdk.IndexStats stats; - - do - { - await Task.Delay(DelayInterval); - attemptCount++; - stats = await this.Index.DescribeStats(); - } while (stats.TotalVectorCount != vectorCountBefore + delta && attemptCount <= MaxAttemptCount); - - if (stats.TotalVectorCount != vectorCountBefore + delta) - { - throw new InvalidOperationException("'Upsert'/'Delete' operation didn't complete in time."); - } - - return stats.TotalVectorCount; - } - - public async Task DeleteAndWaitAsync(IEnumerable ids, string? indexNamespace = null) - { - var stats = await this.Index.DescribeStats(); - var vectorCountBefore = stats.Namespaces.Single(x => x.Name == (indexNamespace ?? "")).VectorCount; - var idCount = ids.Count(); - - var attemptCount = 0; - await this.Index.Delete(ids, indexNamespace); - long vectorCount; - do - { - await Task.Delay(DelayInterval); - attemptCount++; - stats = await this.Index.DescribeStats(); - vectorCount = stats.Namespaces.Single(x => x.Name == (indexNamespace ?? "")).VectorCount; - } while (vectorCount > vectorCountBefore - idCount && attemptCount <= MaxAttemptCount); - - if (vectorCount > vectorCountBefore - idCount) - { - throw new InvalidOperationException("'Delete' operation didn't complete in time."); - } - } - - private async Task ClearIndexesAsync() - { - var indexes = await this.Client.ListIndexes(); - var deletions = indexes.Select(x => this.DeleteExistingIndexAndWaitAsync(x.Name)); - - await Task.WhenAll(deletions); - } - - private async Task DeleteExistingIndexAndWaitAsync(string indexName) - { - var exists = true; - try - { - var attemptCount = 0; - await this.Client.DeleteIndex(indexName); - - do - { - await Task.Delay(DelayInterval); - var indexes = (await this.Client.ListIndexes()).Select(x => x.Name).ToArray(); - if (indexes.Length == 0 || !indexes.Contains(indexName)) - { - exists = false; - } - } while (exists && attemptCount <= MaxAttemptCount); - } - catch (HttpRequestException ex) when (ex.Message.Contains("NOT_FOUND")) - { - // index was already deleted - exists = false; - } - - if (exists) - { - throw new InvalidOperationException("'Delete index' operation didn't complete in time. Index name: " + indexName); - } - } -} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreRecordCollectionTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreRecordCollectionTests.cs deleted file mode 100644 index 9b68eaf8d863..000000000000 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreRecordCollectionTests.cs +++ /dev/null @@ -1,684 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Threading.Tasks; -using Grpc.Core; -using Microsoft.Extensions.VectorData; -using Microsoft.SemanticKernel.Connectors.Pinecone; -using Pinecone; -using SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone.Xunit; -using SemanticKernel.IntegrationTests.Connectors.Memory.Xunit; -using Xunit; - -namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone; - -#pragma warning disable CS0618 // VectorSearchFilter is obsolete - -[Collection("PineconeVectorStoreTests")] -[PineconeApiKeySetCondition] -public class PineconeVectorStoreRecordCollectionTests(PineconeVectorStoreFixture fixture) : IClassFixture -{ - private PineconeVectorStoreFixture Fixture { get; } = fixture; - - [VectorStoreFact] - public async Task TryCreateExistingIndexIsNoopAsync() - { - await this.Fixture.HotelRecordCollection.CreateCollectionIfNotExistsAsync(); - } - - [VectorStoreFact] - public async Task CollectionExistsReturnsTrueForExistingCollectionAsync() - { - var result = await this.Fixture.HotelRecordCollection.CollectionExistsAsync(); - - Assert.True(result); - } - - [VectorStoreTheory] - [InlineData(true)] - [InlineData(false)] - public async Task BasicGetAsync(bool includeVectors) - { - var fiveSeasons = await this.Fixture.HotelRecordCollection.GetAsync("five-seasons", new GetRecordOptions { IncludeVectors = includeVectors }); - - Assert.NotNull(fiveSeasons); - Assert.Equal("five-seasons", fiveSeasons.HotelId); - Assert.Equal("Five Seasons Hotel", fiveSeasons.HotelName); - Assert.Equal("Great service any season.", fiveSeasons.Description); - Assert.Equal(7, fiveSeasons.HotelCode); - Assert.Equal(4.5f, fiveSeasons.HotelRating); - Assert.True(fiveSeasons.ParkingIncluded); - Assert.Contains("wi-fi", fiveSeasons.Tags); - Assert.Contains("sauna", fiveSeasons.Tags); - Assert.Contains("gym", fiveSeasons.Tags); - Assert.Contains("pool", fiveSeasons.Tags); - - if (includeVectors) - { - Assert.Equal(new ReadOnlyMemory([7.5f, 71.0f, 71.5f, 72.0f, 72.5f, 73.0f, 73.5f, 74.0f]), fiveSeasons.DescriptionEmbedding); - } - else - { - Assert.Equal(new ReadOnlyMemory([]), fiveSeasons.DescriptionEmbedding); - } - } - - [VectorStoreTheory] - [InlineData(true)] - [InlineData(false)] - public async Task BatchGetAsync(bool collectionFromVectorStore) - { - var hotelsCollection = collectionFromVectorStore - ? this.Fixture.HotelRecordCollection - : this.Fixture.HotelRecordCollectionFromVectorStore; - - var hotels = await hotelsCollection.GetBatchAsync(["five-seasons", "vacation-inn", "best-eastern"]).ToListAsync(); - - var fiveSeasons = hotels.Single(x => x.HotelId == "five-seasons"); - var vacationInn = hotels.Single(x => x.HotelId == "vacation-inn"); - var bestEastern = hotels.Single(x => x.HotelId == "best-eastern"); - - Assert.Equal("Five Seasons Hotel", fiveSeasons.HotelName); - Assert.Equal("Great service any season.", fiveSeasons.Description); - Assert.Equal(7, fiveSeasons.HotelCode); - Assert.Equal(4.5f, fiveSeasons.HotelRating); - Assert.True(fiveSeasons.ParkingIncluded); - Assert.Contains("wi-fi", fiveSeasons.Tags); - Assert.Contains("sauna", fiveSeasons.Tags); - Assert.Contains("gym", fiveSeasons.Tags); - Assert.Contains("pool", fiveSeasons.Tags); - - Assert.Equal("Vacation Inn Hotel", vacationInn.HotelName); - Assert.Equal("On vacation? Stay with us.", vacationInn.Description); - Assert.Equal(11, vacationInn.HotelCode); - Assert.Equal(4.3f, vacationInn.HotelRating); - Assert.True(vacationInn.ParkingIncluded); - Assert.Contains("wi-fi", vacationInn.Tags); - Assert.Contains("breakfast", vacationInn.Tags); - Assert.Contains("gym", vacationInn.Tags); - - Assert.Equal("Best Eastern Hotel", bestEastern.HotelName); - Assert.Equal("Best hotel east of New York.", bestEastern.Description); - Assert.Equal(42, bestEastern.HotelCode); - Assert.Equal(4.7f, bestEastern.HotelRating); - Assert.True(bestEastern.ParkingIncluded); - Assert.Contains("wi-fi", bestEastern.Tags); - Assert.Contains("breakfast", bestEastern.Tags); - Assert.Contains("gym", bestEastern.Tags); - } - - [VectorStoreTheory] - [InlineData(true)] - [InlineData(false)] - public async Task AllTypesBatchGetAsync(bool includeVectors) - { - var allTypes = await this.Fixture.AllTypesRecordCollection.GetBatchAsync(["all-types-1", "all-types-2"], new GetRecordOptions { IncludeVectors = includeVectors }).ToListAsync(); - - var allTypes1 = allTypes.Single(x => x.Id == "all-types-1"); - var allTypes2 = allTypes.Single(x => x.Id == "all-types-2"); - - Assert.True(allTypes1.BoolProperty); - Assert.Equal("string prop 1", allTypes1.StringProperty); - Assert.Equal(1, allTypes1.IntProperty); - Assert.Equal(100L, allTypes1.LongProperty); - Assert.Equal(10.5f, allTypes1.FloatProperty); - Assert.Equal(23.75d, allTypes1.DoubleProperty); - Assert.Equal(50.75m, allTypes1.DecimalProperty); - Assert.Contains("one", allTypes1.StringArray); - Assert.Contains("two", allTypes1.StringArray); - Assert.Contains("eleven", allTypes1.StringList); - Assert.Contains("twelve", allTypes1.StringList); - Assert.Contains("Foo", allTypes1.Collection); - Assert.Contains("Bar", allTypes1.Collection); - Assert.Contains("another", allTypes1.Enumerable); - Assert.Contains("and another", allTypes1.Enumerable); - - Assert.False(allTypes2.BoolProperty); - Assert.Equal("string prop 2", allTypes2.StringProperty); - Assert.Equal(2, allTypes2.IntProperty); - Assert.Equal(200L, allTypes2.LongProperty); - Assert.Equal(20.5f, allTypes2.FloatProperty); - Assert.Equal(43.75d, allTypes2.DoubleProperty); - Assert.Equal(250.75m, allTypes2.DecimalProperty); - Assert.Empty(allTypes2.StringArray); - Assert.Empty(allTypes2.StringList); - Assert.Empty(allTypes2.Collection); - Assert.Empty(allTypes2.Enumerable); - - if (includeVectors) - { - Assert.True(allTypes1.Embedding.HasValue); - Assert.Equal(new ReadOnlyMemory([1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f, 8.5f]), allTypes1.Embedding.Value); - - Assert.True(allTypes2.Embedding.HasValue); - Assert.Equal(new ReadOnlyMemory([10.5f, 20.5f, 30.5f, 40.5f, 50.5f, 60.5f, 70.5f, 80.5f]), allTypes2.Embedding.Value); - } - else - { - Assert.Null(allTypes1.Embedding); - Assert.Null(allTypes2.Embedding); - } - } - - [VectorStoreFact] - public async Task BatchGetIncludingNonExistingRecordAsync() - { - var hotels = await this.Fixture.HotelRecordCollection.GetBatchAsync(["vacation-inn", "non-existing"]).ToListAsync(); - - Assert.Single(hotels); - var vacationInn = hotels.Single(x => x.HotelId == "vacation-inn"); - - Assert.Equal("Vacation Inn Hotel", vacationInn.HotelName); - Assert.Equal("On vacation? Stay with us.", vacationInn.Description); - Assert.Equal(11, vacationInn.HotelCode); - Assert.Equal(4.3f, vacationInn.HotelRating); - Assert.True(vacationInn.ParkingIncluded); - Assert.Contains("wi-fi", vacationInn.Tags); - Assert.Contains("breakfast", vacationInn.Tags); - Assert.Contains("gym", vacationInn.Tags); - } - - [VectorStoreFact] - public async Task GetNonExistingRecordAsync() - { - var result = await this.Fixture.HotelRecordCollection.GetAsync("non-existing"); - Assert.Null(result); - } - - [VectorStoreTheory] - [InlineData(true)] - [InlineData(false)] - public async Task GetFromCustomNamespaceAsync(bool includeVectors) - { - var custom = await this.Fixture.HotelRecordCollectionWithCustomNamespace.GetAsync("custom-hotel", new GetRecordOptions { IncludeVectors = includeVectors }); - - Assert.NotNull(custom); - Assert.Equal("custom-hotel", custom.HotelId); - Assert.Equal("Custom Hotel", custom.HotelName); - if (includeVectors) - { - Assert.Equal(new ReadOnlyMemory([147.5f, 1421.0f, 1741.5f, 1744.0f, 1742.5f, 1483.0f, 1743.5f, 1744.0f]), custom.DescriptionEmbedding); - } - else - { - Assert.Equal(new ReadOnlyMemory([]), custom.DescriptionEmbedding); - } - } - - [VectorStoreFact] - public async Task TryGetVectorLocatedInDefaultNamespaceButLookInCustomNamespaceAsync() - { - var badFiveSeasons = await this.Fixture.HotelRecordCollectionWithCustomNamespace.GetAsync("five-seasons"); - - Assert.Null(badFiveSeasons); - } - - [VectorStoreFact] - public async Task TryGetVectorLocatedInCustomNamespaceButLookInDefaultNamespaceAsync() - { - var badCustomHotel = await this.Fixture.HotelRecordCollection.GetAsync("custom-hotel"); - - Assert.Null(badCustomHotel); - } - - [VectorStoreFact] - public async Task DeleteNonExistingRecordAsync() - { - await this.Fixture.HotelRecordCollection.DeleteAsync("non-existing"); - } - - [VectorStoreFact] - public async Task TryDeleteExistingVectorLocatedInDefaultNamespaceButUseCustomNamespaceDoesNotDoAnythingAsync() - { - await this.Fixture.HotelRecordCollectionWithCustomNamespace.DeleteAsync("five-seasons"); - - var stillThere = await this.Fixture.HotelRecordCollection.GetAsync("five-seasons"); - Assert.NotNull(stillThere); - Assert.Equal("five-seasons", stillThere.HotelId); - } - - [VectorStoreFact] - public async Task TryDeleteExistingVectorLocatedInCustomNamespaceButUseDefaultNamespaceDoesNotDoAnythingAsync() - { - await this.Fixture.HotelRecordCollection.DeleteAsync("custom-hotel"); - - var stillThere = await this.Fixture.HotelRecordCollectionWithCustomNamespace.GetAsync("custom-hotel"); - Assert.NotNull(stillThere); - Assert.Equal("custom-hotel", stillThere.HotelId); - } - - [VectorStoreTheory] - [InlineData(true)] - [InlineData(false)] - public async Task InsertGetModifyDeleteVectorAsync(bool collectionFromVectorStore) - { - var langriSha = new PineconeHotel - { - HotelId = "langri-sha", - HotelName = "Langri-Sha Hotel", - Description = "Lorem ipsum", - HotelCode = 100, - HotelRating = 4.2f, - ParkingIncluded = false, - DescriptionEmbedding = new ReadOnlyMemory([1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f]) - }; - - var stats = await this.Fixture.Index.DescribeStats(); - var vectorCountBefore = stats.TotalVectorCount; - - var hotelRecordCollection = collectionFromVectorStore - ? this.Fixture.HotelRecordCollectionFromVectorStore - : this.Fixture.HotelRecordCollection; - - // insert - await hotelRecordCollection.UpsertAsync(langriSha); - - vectorCountBefore = await this.Fixture.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: 1); - - var inserted = await hotelRecordCollection.GetAsync("langri-sha", new GetRecordOptions { IncludeVectors = true }); - - Assert.NotNull(inserted); - Assert.Equal(langriSha.HotelName, inserted.HotelName); - Assert.Equal(langriSha.Description, inserted.Description); - Assert.Equal(langriSha.HotelCode, inserted.HotelCode); - Assert.Equal(langriSha.HotelRating, inserted.HotelRating); - Assert.Equal(langriSha.ParkingIncluded, inserted.ParkingIncluded); - Assert.Equal(langriSha.DescriptionEmbedding, inserted.DescriptionEmbedding); - - langriSha.Description += " dolor sit amet"; - langriSha.ParkingIncluded = true; - langriSha.DescriptionEmbedding = new ReadOnlyMemory([11f, 12f, 13f, 14f, 15f, 16f, 17f, 18f]); - - // update - await hotelRecordCollection.UpsertAsync(langriSha); - - // this is not great but no vectors are added so we can't query status for number of vectors like we do for insert/delete - await Task.Delay(2000); - - var updated = await hotelRecordCollection.GetAsync("langri-sha", new GetRecordOptions { IncludeVectors = true }); - - Assert.NotNull(updated); - Assert.Equal(langriSha.HotelName, updated.HotelName); - Assert.Equal(langriSha.Description, updated.Description); - Assert.Equal(langriSha.HotelCode, updated.HotelCode); - Assert.Equal(langriSha.HotelRating, updated.HotelRating); - Assert.Equal(langriSha.ParkingIncluded, updated.ParkingIncluded); - Assert.Equal(langriSha.DescriptionEmbedding, updated.DescriptionEmbedding); - - // delete - await hotelRecordCollection.DeleteAsync("langri-sha"); - - await this.Fixture.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: -1); - } - - [VectorStoreTheory] - [InlineData(true, true)] - [InlineData(true, false)] - [InlineData(false, true)] - [InlineData(false, false)] - public async Task VectorizedSearchAsync(bool collectionFromVectorStore, bool includeVectors) - { - // Arrange. - var hotelRecordCollection = collectionFromVectorStore - ? this.Fixture.HotelRecordCollectionFromVectorStore - : this.Fixture.HotelRecordCollection; - var searchVector = new ReadOnlyMemory([17.5f, 721.0f, 731.5f, 742.0f, 762.5f, 783.0f, 793.5f, 704.0f]); - - // Act. - var actual = await hotelRecordCollection.VectorizedSearchAsync(searchVector, new() { IncludeVectors = includeVectors }); - var searchResults = await actual.Results.ToListAsync(); - var searchResultRecord = searchResults.First().Record; - - Assert.Equal("Vacation Inn Hotel", searchResultRecord.HotelName); - Assert.Equal("On vacation? Stay with us.", searchResultRecord.Description); - Assert.Equal(11, searchResultRecord.HotelCode); - Assert.Equal(4.3f, searchResultRecord.HotelRating); - Assert.True(searchResultRecord.ParkingIncluded); - Assert.Contains("wi-fi", searchResultRecord.Tags); - Assert.Contains("breakfast", searchResultRecord.Tags); - Assert.Contains("gym", searchResultRecord.Tags); - Assert.Equal(includeVectors, searchResultRecord.DescriptionEmbedding.Length > 0); - } - - [VectorStoreTheory] - [InlineData(true)] - [InlineData(false)] - public async Task VectorizedSearchWithTopSkipAsync(bool collectionFromVectorStore) - { - // Arrange. - var hotelRecordCollection = collectionFromVectorStore - ? this.Fixture.HotelRecordCollectionFromVectorStore - : this.Fixture.HotelRecordCollection; - var searchVector = new ReadOnlyMemory([17.5f, 721.0f, 731.5f, 742.0f, 762.5f, 783.0f, 793.5f, 704.0f]); - - // Act. - var actual = await hotelRecordCollection.VectorizedSearchAsync(searchVector, new() { Skip = 1, Top = 1 }); - var searchResults = await actual.Results.ToListAsync(); - Assert.Single(searchResults); - var searchResultRecord = searchResults.First().Record; - Assert.Equal("Best Eastern Hotel", searchResultRecord.HotelName); - } - - [VectorStoreTheory] - [InlineData(true)] - [InlineData(false)] - public async Task VectorizedSearchWithFilterAsync(bool collectionFromVectorStore) - { - // Arrange. - var hotelRecordCollection = collectionFromVectorStore - ? this.Fixture.HotelRecordCollectionFromVectorStore - : this.Fixture.HotelRecordCollection; - var searchVector = new ReadOnlyMemory([17.5f, 721.0f, 731.5f, 742.0f, 762.5f, 783.0f, 793.5f, 704.0f]); - - // Act. - var filter = new VectorSearchFilter().EqualTo(nameof(PineconeHotel.HotelCode), 42); - var actual = await hotelRecordCollection.VectorizedSearchAsync(searchVector, new() { Top = 1, OldFilter = filter }); - var searchResults = await actual.Results.ToListAsync(); - Assert.Single(searchResults); - var searchResultRecord = searchResults.First().Record; - Assert.Equal("Best Eastern Hotel", searchResultRecord.HotelName); - } - - [VectorStoreFact] - public async Task ItCanUpsertAndRetrieveUsingTheGenericMapperAsync() - { - var merryYacht = new VectorStoreGenericDataModel("merry-yacht") - { - Data = - { - ["HotelName"] = "Merry Yacht Hotel", - ["Description"] = "Stay afloat at the Merry Yacht Hotel", - ["HotelCode"] = 101, - ["HotelRating"] = 4.2f, - ["ParkingIncluded"] = true, - ["Tags"] = new[] { "wi-fi", "breakfast", "gym" } - }, - Vectors = - { - ["DescriptionEmbedding"] = new ReadOnlyMemory([1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f]) - } - }; - - var stats = await this.Fixture.Index.DescribeStats(); - var vectorCountBefore = stats.TotalVectorCount; - - var hotelRecordCollection = this.Fixture.HotelRecordCollectionWithGenericDataModel; - - // insert - await hotelRecordCollection.UpsertAsync(merryYacht); - - vectorCountBefore = await this.Fixture.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: 1); - - var inserted = await hotelRecordCollection.GetAsync("merry-yacht", new GetRecordOptions { IncludeVectors = true }); - - Assert.NotNull(inserted); - Assert.Equal(merryYacht.Data["HotelName"], inserted.Data["HotelName"]); - Assert.Equal(merryYacht.Data["Description"], inserted.Data["Description"]); - Assert.Equal(merryYacht.Data["HotelCode"], inserted.Data["HotelCode"]); - Assert.Equal(merryYacht.Data["HotelRating"], inserted.Data["HotelRating"]); - Assert.Equal(merryYacht.Data["ParkingIncluded"], inserted.Data["ParkingIncluded"]); - Assert.Equal(merryYacht.Data["Tags"], inserted.Data["Tags"]); - Assert.Equal( - ((ReadOnlyMemory)merryYacht.Vectors["DescriptionEmbedding"]!).ToArray(), - ((ReadOnlyMemory)inserted.Vectors["DescriptionEmbedding"]!).ToArray()); - - // delete - await hotelRecordCollection.DeleteAsync("merry-yacht"); - - await this.Fixture.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: -1); - } - - [VectorStoreFact] - public async Task UseCollectionExistsOnNonExistingStoreReturnsFalseAsync() - { - var incorrectRecordStore = new PineconeVectorStoreRecordCollection( - this.Fixture.Client, - "incorrect"); - - var result = await incorrectRecordStore.CollectionExistsAsync(); - - Assert.False(result); - } - - [VectorStoreFact] - public async Task UseNonExistingIndexThrowsAsync() - { - var incorrectRecordStore = new PineconeVectorStoreRecordCollection( - this.Fixture.Client, - "incorrect"); - - var statusCode = (await Assert.ThrowsAsync( - () => incorrectRecordStore.GetAsync("best-eastern"))).StatusCode; - - Assert.Equal(HttpStatusCode.NotFound, statusCode); - } - - [VectorStoreFact] - public async Task UseRecordStoreWithCustomMapperAsync() - { - var recordStore = new PineconeVectorStoreRecordCollection( - this.Fixture.Client, - this.Fixture.IndexName, - new PineconeVectorStoreRecordCollectionOptions { VectorCustomMapper = new CustomHotelRecordMapper() }); - - var vacationInn = await recordStore.GetAsync("vacation-inn", new GetRecordOptions { IncludeVectors = true }); - - Assert.NotNull(vacationInn); - Assert.Equal("Custom Vacation Inn Hotel", vacationInn.HotelName); - Assert.Equal("On vacation? Stay with us.", vacationInn.Description); - Assert.Equal(11, vacationInn.HotelCode); - Assert.Equal(4.3f, vacationInn.HotelRating); - Assert.True(vacationInn.ParkingIncluded); - Assert.Contains("wi-fi", vacationInn.Tags); - Assert.Contains("breakfast", vacationInn.Tags); - Assert.Contains("gym", vacationInn.Tags); - } - - private sealed class CustomHotelRecordMapper : IVectorStoreRecordMapper - { - public Vector MapFromDataToStorageModel(PineconeHotel dataModel) - { - var metadata = new MetadataMap - { - [nameof(PineconeHotel.HotelName)] = dataModel.HotelName, - [nameof(PineconeHotel.Description)] = dataModel.Description, - [nameof(PineconeHotel.HotelCode)] = dataModel.HotelCode, - [nameof(PineconeHotel.HotelRating)] = dataModel.HotelRating, - ["parking_is_included"] = dataModel.ParkingIncluded, - [nameof(PineconeHotel.Tags)] = dataModel.Tags.ToArray(), - }; - - return new Vector - { - Id = dataModel.HotelId, - Values = dataModel.DescriptionEmbedding.ToArray(), - Metadata = metadata, - }; - } - - public PineconeHotel MapFromStorageToDataModel(Vector storageModel, StorageToDataModelMapperOptions options) - { - if (storageModel.Metadata == null) - { - throw new InvalidOperationException("Missing metadata."); - } - - return new PineconeHotel - { - HotelId = storageModel.Id, - HotelName = "Custom " + (string)storageModel.Metadata[nameof(PineconeHotel.HotelName)].Inner!, - Description = (string)storageModel.Metadata[nameof(PineconeHotel.Description)].Inner!, - HotelCode = (int)(double)storageModel.Metadata[nameof(PineconeHotel.HotelCode)].Inner!, - HotelRating = (float)(double)storageModel.Metadata[nameof(PineconeHotel.HotelRating)].Inner!, - ParkingIncluded = (bool)storageModel.Metadata["parking_is_included"].Inner!, - Tags = ((MetadataValue[])storageModel.Metadata[nameof(PineconeHotel.Tags)].Inner!)!.Select(x => (string)x.Inner!).ToList(), - }; - } - } - - #region Negative - - [VectorStoreFact] - public void UseRecordWithNoEmbeddingThrows() - { - var exception = Assert.Throws( - () => new PineconeVectorStoreRecordCollection( - this.Fixture.Client, - "Whatever")); - - Assert.Equal( - $"No vector property found on type {nameof(PineconeRecordNoEmbedding)} or the provided VectorStoreRecordDefinition while at least one is required.", - exception.Message); - } - -#pragma warning disable CA1812 - private sealed record PineconeRecordNoEmbedding - { - [VectorStoreRecordKey] - public int Id { get; set; } - - [VectorStoreRecordData] - public string? Name { get; set; } - } -#pragma warning restore CA1812 - - [VectorStoreFact] - public void UseRecordWithMultipleEmbeddingsThrows() - { - var exception = Assert.Throws( - () => new PineconeVectorStoreRecordCollection( - this.Fixture.Client, - "Whatever")); - - Assert.Equal( - $"Multiple vector properties found on type {nameof(PineconeRecordMultipleEmbeddings)} or the provided VectorStoreRecordDefinition while only one is supported.", - exception.Message); - } - -#pragma warning disable CA1812 - private sealed record PineconeRecordMultipleEmbeddings - { - [VectorStoreRecordKey] - public string Id { get; set; } = null!; - - [VectorStoreRecordVector] - public ReadOnlyMemory Embedding1 { get; set; } - - [VectorStoreRecordVector] - public ReadOnlyMemory Embedding2 { get; set; } - } -#pragma warning restore CA1812 - - [VectorStoreFact] - public void UseRecordWithUnsupportedKeyTypeThrows() - { - var message = Assert.Throws( - () => new PineconeVectorStoreRecordCollection( - this.Fixture.Client, - "Whatever")).Message; - - Assert.Equal( - $"Key properties must be one of the supported types: {typeof(string).FullName}. Type of the property '{nameof(PineconeRecordUnsupportedKeyType.Id)}' is {typeof(int).FullName}.", - message); - } - -#pragma warning disable CA1812 - private sealed record PineconeRecordUnsupportedKeyType - { - [VectorStoreRecordKey] - public int Id { get; set; } - - [VectorStoreRecordData] - public string? Name { get; set; } - - [VectorStoreRecordVector] - public ReadOnlyMemory Embedding { get; set; } - } -#pragma warning restore CA1812 - - [VectorStoreFact] - public async Task TryAddingVectorWithUnsupportedValuesAsync() - { - var badAllTypes = new PineconeAllTypes - { - Id = "bad", - BoolProperty = true, - DecimalProperty = 1m, - DoubleProperty = 1.5d, - FloatProperty = 2.5f, - IntProperty = 1, - LongProperty = 11L, - NullableStringArray = ["foo", null!, "bar",], - Embedding = new ReadOnlyMemory([1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f]) - }; - - var exception = await Assert.ThrowsAsync( - () => this.Fixture.AllTypesRecordCollection.UpsertAsync(badAllTypes)); - - Assert.Equal("Microsoft.SemanticKernel.Connectors.Pinecone", exception.Source); - Assert.Equal("Pinecone", exception.VectorStoreType); - Assert.Equal("Upsert", exception.OperationName); - Assert.Equal(this.Fixture.IndexName, exception.CollectionName); - - var inner = exception.InnerException as RpcException; - Assert.NotNull(inner); - Assert.Equal(StatusCode.InvalidArgument, inner.StatusCode); - } - - [VectorStoreFact] - public async Task TryCreateIndexWithIncorrectDimensionFailsAsync() - { - var recordCollection = new PineconeVectorStoreRecordCollection( - this.Fixture.Client, - "negative-dimension"); - - var message = (await Assert.ThrowsAsync(() => recordCollection.CreateCollectionAsync())).Message; - - Assert.Equal("Property Dimensions on VectorStoreRecordVectorProperty 'Embedding' must be set to a positive integer to create a collection.", message); - } - -#pragma warning disable CA1812 - private sealed record PineconeRecordWithIncorrectDimension - { - [VectorStoreRecordKey] - public string Id { get; set; } = null!; - - [VectorStoreRecordData] - public string? Name { get; set; } - - [VectorStoreRecordVector(Dimensions: -7)] - public ReadOnlyMemory Embedding { get; set; } - } -#pragma warning restore CA1812 - - [VectorStoreFact] - public async Task TryCreateIndexWithUnsSupportedMetricFailsAsync() - { - var recordCollection = new PineconeVectorStoreRecordCollection( - this.Fixture.Client, - "bad-metric"); - - var message = (await Assert.ThrowsAsync(() => recordCollection.CreateCollectionAsync())).Message; - - Assert.Equal("Distance function 'just eyeball it' for VectorStoreRecordVectorProperty 'Embedding' is not supported by the Pinecone VectorStore.", message); - } - -#pragma warning disable CA1812 - private sealed record PineconeRecordWithUnsupportedMetric - { - [VectorStoreRecordKey] - public string Id { get; set; } = null!; - - [VectorStoreRecordData] - public string? Name { get; set; } - - [VectorStoreRecordVector(Dimensions: 5, DistanceFunction: "just eyeball it")] - public ReadOnlyMemory Embedding { get; set; } - } -#pragma warning restore CA1812 - - #endregion -} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreTests.cs deleted file mode 100644 index 4cd63ec6b8a9..000000000000 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreTests.cs +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.Extensions.VectorData; -using Microsoft.SemanticKernel.Connectors.Pinecone; -using SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone.Xunit; -using SemanticKernel.IntegrationTests.Connectors.Memory.Xunit; -using Xunit; -using Sdk = Pinecone; - -namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone; - -[Collection("PineconeVectorStoreTests")] -[PineconeApiKeySetCondition] -public class PineconeVectorStoreTests(PineconeVectorStoreFixture fixture) - : BaseVectorStoreTests(new PineconeVectorStore(fixture.Client)), IClassFixture -{ - private PineconeVectorStoreFixture Fixture { get; } = fixture; - -#pragma warning disable CS0618 // IPineconeVectorStoreRecordCollectionFactory is obsolete - [VectorStoreFact] - public void CreateCollectionUsingFactory() - { - var vectorStore = new PineconeVectorStore( - this.Fixture.Client, - new PineconeVectorStoreOptions - { - VectorStoreCollectionFactory = new MyVectorStoreRecordCollectionFactory() - }); - - var factoryCollection = vectorStore.GetCollection(this.Fixture.IndexName); - - Assert.NotNull(factoryCollection); - Assert.Equal("factory" + this.Fixture.IndexName, factoryCollection.CollectionName); - } - - private sealed class MyVectorStoreRecordCollectionFactory : IPineconeVectorStoreRecordCollectionFactory - { - public IVectorStoreRecordCollection CreateVectorStoreRecordCollection( - Sdk.PineconeClient pineconeClient, - string name, - VectorStoreRecordDefinition? vectorStoreRecordDefinition) - where TKey : notnull - { - if (typeof(TKey) != typeof(string)) - { - throw new InvalidOperationException("Only string keys are supported."); - } - - return (new PineconeVectorStoreRecordCollection(pineconeClient, "factory" + name) as IVectorStoreRecordCollection)!; - } - } -#pragma warning restore CS0618 -} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/Xunit/PineconeApiKeySetConditionAttribute.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/Xunit/PineconeApiKeySetConditionAttribute.cs deleted file mode 100644 index b677c47c378f..000000000000 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/Xunit/PineconeApiKeySetConditionAttribute.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading.Tasks; -using SemanticKernel.IntegrationTests.Connectors.Memory.Xunit; - -namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone.Xunit; - -[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class)] -public sealed class PineconeApiKeySetConditionAttribute : Attribute, ITestCondition -{ - public ValueTask IsMetAsync() - { - var isMet = PineconeUserSecretsExtensions.ContainsPineconeApiKey(); - - return ValueTask.FromResult(isMet); - } - - public string SkipReason - => $"Pinecone API key was not specified in user secrets. Use the following command to set it: dotnet user-secrets set \"{PineconeUserSecretsExtensions.PineconeApiKeyUserSecretEntry}\" \"your_Pinecone_API_key\""; -} diff --git a/dotnet/src/IntegrationTests/IntegrationTests.csproj b/dotnet/src/IntegrationTests/IntegrationTests.csproj index 9f4e835df255..917e2447fea3 100644 --- a/dotnet/src/IntegrationTests/IntegrationTests.csproj +++ b/dotnet/src/IntegrationTests/IntegrationTests.csproj @@ -53,7 +53,6 @@ - @@ -70,7 +69,6 @@ - diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeAllSupportedTypesTests.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeAllSupportedTypesTests.cs new file mode 100644 index 000000000000..e1dbf9efa81e --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeAllSupportedTypesTests.cs @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.VectorData; +using PineconeIntegrationTests.Support; +using VectorDataSpecificationTests.Xunit; +using Xunit; + +namespace PineconeIntegrationTests.CRUD; + +public class PineconeAllSupportedTypesTests(PineconeFixture fixture) : IClassFixture +{ + [ConditionalFact] + public async Task AllTypesBatchGetAsync() + { + var collection = fixture.TestStore.DefaultVectorStore.GetCollection("all-types", PineconeAllTypes.GetRecordDefinition()); + await collection.CreateCollectionIfNotExistsAsync(); + + List records = + [ + new() + { + Id = "all-types-1", + BoolProperty = true, + NullableBoolProperty = false, + StringProperty = "string prop 1", + NullableStringProperty = "nullable prop 1", + IntProperty = 1, + NullableIntProperty = 10, + LongProperty = 100L, + NullableLongProperty = 1000L, + FloatProperty = 10.5f, + NullableFloatProperty = 100.5f, + DoubleProperty = 23.75d, + NullableDoubleProperty = 233.75d, + StringArray = ["one", "two"], + NullableStringArray = ["five", "six"], + StringList = ["eleven", "twelve"], + NullableStringList = ["fifteen", "sixteen"], + Embedding = new ReadOnlyMemory([1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f, 8.5f]) + }, + new() + { + Id = "all-types-2", + BoolProperty = false, + NullableBoolProperty = null, + StringProperty = "string prop 2", + NullableStringProperty = null, + IntProperty = 2, + NullableIntProperty = null, + LongProperty = 200L, + NullableLongProperty = null, + FloatProperty = 20.5f, + NullableFloatProperty = null, + DoubleProperty = 43.75, + NullableDoubleProperty = null, + StringArray = [], + NullableStringArray = null, + StringList = [], + NullableStringList = null, + Embedding = new ReadOnlyMemory([10.5f, 20.5f, 30.5f, 40.5f, 50.5f, 60.5f, 70.5f, 80.5f]) + } + ]; + + await collection.UpsertBatchAsync(records).ToArrayAsync(); + + var allTypes = await collection.GetBatchAsync(records.Select(r => r.Id), new GetRecordOptions { IncludeVectors = true }).ToListAsync(); + + var allTypes1 = allTypes.Single(x => x.Id == records[0].Id); + var allTypes2 = allTypes.Single(x => x.Id == records[1].Id); + + records[0].AssertEqual(allTypes1); + records[1].AssertEqual(allTypes2); + } +} diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeBatchConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeBatchConformanceTests.cs new file mode 100644 index 000000000000..16832369b26d --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeBatchConformanceTests.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. + +using PineconeIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using Xunit; + +namespace PineconeIntegrationTests.CRUD; + +public class PineconeBatchConformanceTests(PineconeSimpleModelFixture fixture) + : BatchConformanceTests(fixture), IClassFixture +{ +} diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeGenericDataModelConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeGenericDataModelConformanceTests.cs new file mode 100644 index 000000000000..d18cdb99b38f --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeGenericDataModelConformanceTests.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. + +using PineconeIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using Xunit; + +namespace PineconeIntegrationTests.CRUD; + +public class PineconeGenericDataModelConformanceTests(PineconeGenericDataModelFixture fixture) + : GenericDataModelConformanceTests(fixture), IClassFixture +{ +} diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeRecordConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeRecordConformanceTests.cs new file mode 100644 index 000000000000..6721ea6dd359 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeRecordConformanceTests.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. + +using PineconeIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using Xunit; + +namespace PineconeIntegrationTests.CRUD; + +public class PineconeRecordConformanceTests(PineconeSimpleModelFixture fixture) + : RecordConformanceTests(fixture), IClassFixture +{ +} diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Collections/PineconeCollectionConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Collections/PineconeCollectionConformanceTests.cs new file mode 100644 index 000000000000..2fbed4bfcd11 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Collections/PineconeCollectionConformanceTests.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. + +using PineconeIntegrationTests.Support; +using VectorDataSpecificationTests.Collections; +using Xunit; + +namespace PineconeIntegrationTests.Collections; + +public class PineconeCollectionConformanceTests(PineconeFixture fixture) + : CollectionConformanceTests(fixture), IClassFixture +{ +} diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Filter/PineconeBasicFilterTests.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Filter/PineconeBasicFilterTests.cs new file mode 100644 index 000000000000..095b0d03ebd0 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Filter/PineconeBasicFilterTests.cs @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft. All rights reserved. + +using PineconeIntegrationTests.Support; +using VectorDataSpecificationTests.Filter; +using VectorDataSpecificationTests.Support; +using VectorDataSpecificationTests.Xunit; +using Xunit; + +namespace PineconeIntegrationTests.Filter; + +public class PineconeBasicFilterTests(PineconeBasicFilterTests.Fixture fixture) + : BasicFilterTests(fixture), IClassFixture +{ + // Specialized Pinecone syntax for NOT over Contains ($nin) + [ConditionalFact] + public virtual Task Not_over_Contains() + => this.TestFilterAsync(r => !new[] { 8, 10 }.Contains(r.Int)); + + #region Null checking + + // Pinecone currently doesn't support null checking ({ "Foo" : null }) in vector search pre-filters + public override Task Equal_with_null_reference_type() + => Assert.ThrowsAsync(() => base.Equal_with_null_reference_type()); + + public override Task Equal_with_null_captured() + => Assert.ThrowsAsync(() => base.Equal_with_null_captured()); + + public override Task NotEqual_with_null_reference_type() + => Assert.ThrowsAsync(() => base.NotEqual_with_null_reference_type()); + + public override Task NotEqual_with_null_captured() + => Assert.ThrowsAsync(() => base.NotEqual_with_null_captured()); + + #endregion + + #region Not + + // Pinecone currently doesn't support NOT in vector search pre-filters + // (https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-stage/#atlas-vector-search-pre-filter) + public override Task Not_over_And() + => Assert.ThrowsAsync(() => base.Not_over_And()); + + public override Task Not_over_Or() + => Assert.ThrowsAsync(() => base.Not_over_Or()); + + #endregion + + public override Task Contains_over_field_string_array() + => Assert.ThrowsAsync(() => base.Contains_over_field_string_array()); + + public override Task Contains_over_field_string_List() + => Assert.ThrowsAsync(() => base.Contains_over_field_string_List()); + + // AnyTagEqualTo not (currently) supported on Pinecone + [Obsolete("Legacy filter support")] + public override Task Legacy_AnyTagEqualTo_array() + => Assert.ThrowsAsync(() => base.Legacy_AnyTagEqualTo_array()); + + [Obsolete("Legacy filter support")] + public override Task Legacy_AnyTagEqualTo_List() + => Assert.ThrowsAsync(() => base.Legacy_AnyTagEqualTo_List()); + + public new class Fixture : BasicFilterTests.Fixture + { + public override TestStore TestStore => PineconeTestStore.Instance; + + // https://docs.pinecone.io/troubleshooting/restrictions-on-index-names + protected override string CollectionName => "filter-tests"; + } +} diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/PineconeIntegrationTests.csproj b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/PineconeIntegrationTests.csproj new file mode 100644 index 000000000000..bc92e1816858 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/PineconeIntegrationTests.csproj @@ -0,0 +1,41 @@ + + + + + net8.0 + enable + enable + + false + true + + $(NoWarn);CA2007,SKEXP0001,SKEXP0020,VSTHRD111;CS1685 + b7762d10-e29b-4bb1-8b74-b6d69a667dd4 + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Properties/AssemblyAttributes.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Properties/AssemblyAttributes.cs new file mode 100644 index 000000000000..c94e7a708820 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Properties/AssemblyAttributes.cs @@ -0,0 +1,5 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Xunit; + +[assembly: CollectionBehavior(DisableTestParallelization = true)] diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeAllTypes.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeAllTypes.cs new file mode 100644 index 000000000000..54d98f72c251 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeAllTypes.cs @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.VectorData; +using Xunit; + +namespace PineconeIntegrationTests.Support; + +#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. +public record PineconeAllTypes() +{ + [VectorStoreRecordKey] + public string Id { get; set; } + + [VectorStoreRecordData] + public bool BoolProperty { get; set; } + [VectorStoreRecordData] + public bool? NullableBoolProperty { get; set; } + [VectorStoreRecordData] + public string StringProperty { get; set; } + [VectorStoreRecordData] + public string? NullableStringProperty { get; set; } + [VectorStoreRecordData] + public int IntProperty { get; set; } + [VectorStoreRecordData] + public int? NullableIntProperty { get; set; } + [VectorStoreRecordData] + public long LongProperty { get; set; } + [VectorStoreRecordData] + public long? NullableLongProperty { get; set; } + [VectorStoreRecordData] + public float FloatProperty { get; set; } + [VectorStoreRecordData] + public float? NullableFloatProperty { get; set; } + [VectorStoreRecordData] + public double DoubleProperty { get; set; } + [VectorStoreRecordData] + public double? NullableDoubleProperty { get; set; } + +#pragma warning disable CA1819 // Properties should not return arrays + [VectorStoreRecordData] + public string[] StringArray { get; set; } + [VectorStoreRecordData] + public string[]? NullableStringArray { get; set; } +#pragma warning restore CA1819 // Properties should not return arrays + + [VectorStoreRecordData] + public List StringList { get; set; } + [VectorStoreRecordData] + public List? NullableStringList { get; set; } + + [VectorStoreRecordVector(Dimensions: 8, DistanceFunction: DistanceFunction.DotProductSimilarity)] + public ReadOnlyMemory? Embedding { get; set; } + + internal void AssertEqual(PineconeAllTypes other) + { + Assert.Equal(this.Id, other.Id); + Assert.Equal(this.BoolProperty, other.BoolProperty); + Assert.Equal(this.NullableBoolProperty, other.NullableBoolProperty); + Assert.Equal(this.StringProperty, other.StringProperty); + Assert.Equal(this.NullableStringProperty, other.NullableStringProperty); + Assert.Equal(this.IntProperty, other.IntProperty); + Assert.Equal(this.NullableIntProperty, other.NullableIntProperty); + Assert.Equal(this.LongProperty, other.LongProperty); + Assert.Equal(this.NullableLongProperty, other.NullableLongProperty); + Assert.Equal(this.FloatProperty, other.FloatProperty); + Assert.Equal(this.NullableFloatProperty, other.NullableFloatProperty); + Assert.Equal(this.DoubleProperty, other.DoubleProperty); + Assert.Equal(this.NullableDoubleProperty, other.NullableDoubleProperty); + Assert.Equal(this.StringArray, other.StringArray); + Assert.Equal(this.NullableStringArray, other.NullableStringArray); + Assert.Equal(this.StringList, other.StringList); + Assert.Equal(this.NullableStringList, other.NullableStringList); + Assert.Equal(this.Embedding!.Value.ToArray(), other.Embedding!.Value.ToArray()); + } + + internal static VectorStoreRecordDefinition GetRecordDefinition() + => new() + { + Properties = + [ + new VectorStoreRecordKeyProperty(nameof(PineconeAllTypes.Id), typeof(string)), + new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.BoolProperty), typeof(bool)), + new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableBoolProperty), typeof(bool?)), + new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.StringProperty), typeof(string)), + new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableStringProperty), typeof(string)), + new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.IntProperty), typeof(int)), + new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableIntProperty), typeof(int?)), + new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.LongProperty), typeof(long)), + new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableLongProperty), typeof(long?)), + new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.FloatProperty), typeof(float)), + new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableFloatProperty), typeof(float?)), + new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.DoubleProperty), typeof(double)), + new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableDoubleProperty), typeof(double?)), + new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.StringArray), typeof(string[])), + new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableStringArray), typeof(string[])), + new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.StringList), typeof(List)), + new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableStringList), typeof(List)), + new VectorStoreRecordVectorProperty(nameof(PineconeAllTypes.Embedding), typeof(ReadOnlyMemory?)) { Dimensions = 8, DistanceFunction = Microsoft.Extensions.VectorData.DistanceFunction.DotProductSimilarity } + ] + }; +} +#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeFixture.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeFixture.cs new file mode 100644 index 000000000000..6b3c068b597e --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeFixture.cs @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft. All rights reserved. + +using VectorDataSpecificationTests.Support; + +namespace PineconeIntegrationTests.Support; + +public class PineconeFixture : VectorStoreFixture +{ + public override TestStore TestStore => PineconeTestStore.Instance; +} diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeGenericDataModelFixture.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeGenericDataModelFixture.cs new file mode 100644 index 000000000000..91768966c9ff --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeGenericDataModelFixture.cs @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft. All rights reserved. + +using VectorDataSpecificationTests.Support; + +namespace PineconeIntegrationTests.Support; + +public class PineconeGenericDataModelFixture : GenericDataModelFixture +{ + public override TestStore TestStore => PineconeTestStore.Instance; +} diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeSimpleModelFixture.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeSimpleModelFixture.cs new file mode 100644 index 000000000000..4835b3aa6eeb --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeSimpleModelFixture.cs @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft. All rights reserved. + +using VectorDataSpecificationTests.Support; + +namespace PineconeIntegrationTests.Support; + +public class PineconeSimpleModelFixture : SimpleModelFixture +{ + public override TestStore TestStore => PineconeTestStore.Instance; +} diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeTestStore.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeTestStore.cs new file mode 100644 index 000000000000..40d3e221e777 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeTestStore.cs @@ -0,0 +1,141 @@ +// Copyright (c) Microsoft. All rights reserved. + +#pragma warning disable IDE0005 // Using directive is unnecessary. +using System.Net.Http; +#pragma warning restore IDE0005 // Using directive is unnecessary. +using DotNet.Testcontainers.Builders; +using DotNet.Testcontainers.Containers; +using Microsoft.Extensions.VectorData; +using Microsoft.SemanticKernel.Connectors.Pinecone; +using Pinecone; +using VectorDataSpecificationTests.Support; + +namespace PineconeIntegrationTests.Support; + +#pragma warning disable CA1001 // Type owns disposable fields but is not disposable +#pragma warning disable CA2000 // Dispose objects before losing scope + +internal sealed class PineconeTestStore : TestStore +{ + // Values taken from https://docs.pinecone.io/guides/operations/local-development + // v0.7.0 works with 2.1 client + // v1.0.0 works with 3.0 client + // We use hardcoded version to avoid breaking changes. + private const string Image = "ghcr.io/pinecone-io/pinecone-local:v1.0.0.rc0"; + private const ushort FirstPort = 5080; + private const int IndexServiceCount = 10; + + public static PineconeTestStore Instance { get; } = new(); + + private IContainer? _container; + private Pinecone.PineconeClient? _client; + private PineconeVectorStore? _defaultVectorStore; + + public Pinecone.PineconeClient Client => this._client ?? throw new InvalidOperationException("Not initialized"); + + public override IVectorStore DefaultVectorStore => this._defaultVectorStore ?? throw new InvalidOperationException("Not initialized"); + + // Pinecone does not support distance functions other than PGA which is always enabled. + public override string DefaultIndexKind => ""; + + private PineconeTestStore() + { + } + + protected override async Task StartAsync() + { + this._container = await this.StartContainerAsync(); + + Dictionary containerToHostPort = Enumerable.Range(FirstPort, IndexServiceCount + 1) + .ToDictionary(port => port, port => (int)this._container.GetMappedPublicPort(port)); + + UriBuilder baseAddress = new() + { + Scheme = "http", + Host = this._container.Hostname, + Port = this._container.GetMappedPublicPort(FirstPort) + }; + + ClientOptions clientOptions = new() + { + BaseUrl = baseAddress.Uri.ToString(), + MaxRetries = 0, + IsTlsEnabled = false, + GrpcOptions = new() + { + HttpClient = new(new RedirectHandler(containerToHostPort), disposeHandler: true) + { + BaseAddress = baseAddress.Uri + }, + } + }; + + this._client = new Pinecone.PineconeClient( + apiKey: "ForPineconeLocalTheApiKeysAreIgnored", + clientOptions: clientOptions); + + this._defaultVectorStore = new(this._client); + } + + protected override async Task StopAsync() + { + if (this._container is not null) + { + await this._container.DisposeAsync(); + } + } + + private async Task StartContainerAsync() + { + ContainerBuilder builder = new ContainerBuilder() + .WithImage(Image) + // Pinecone Local will run on port $FirstPort. + .WithPortBinding(FirstPort, assignRandomHostPort: true) + // We are currently using the default Pinecone port (5080), but we can change it to a random port. + // In such case, we are going to need to set the PORT environment variable to the new port. + .WithEnvironment("PORT", FirstPort.ToString()); + + for (int indexService = 1; indexService <= IndexServiceCount; indexService++) + { + // And the index services on the following ports. + builder = builder.WithPortBinding(FirstPort + indexService, assignRandomHostPort: true); + } + + var container = builder.Build(); + + await container.StartAsync(); + + return container; + } + + private sealed class RedirectHandler : DelegatingHandler + { + private readonly Dictionary _containerToHostPort; + + public RedirectHandler(Dictionary portRedirections) + : base(new HttpClientHandler()) + { + this._containerToHostPort = portRedirections; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + // When "host" argument is not provided for PineconeClient.Index, + // it will try to get the host from the Pinecone service. + // In the cloud environment it's fine, but with the local emulator + // it reports the address with the container port, not the host port. + if (request.RequestUri != null && request.RequestUri.IsAbsoluteUri + && request.RequestUri.Host == "localhost" + && this._containerToHostPort.TryGetValue(request.RequestUri.Port, out int hostPort)) + { + UriBuilder builder = new(request.RequestUri) + { + Port = hostPort + }; + request.RequestUri = builder.Uri; + } + + return base.SendAsync(request, cancellationToken); + } + } +} diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/VectorSearch/PineconeVectorSearchDistanceFunctionComplianceTests.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/VectorSearch/PineconeVectorSearchDistanceFunctionComplianceTests.cs new file mode 100644 index 000000000000..2899e0cfbd7d --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/VectorSearch/PineconeVectorSearchDistanceFunctionComplianceTests.cs @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft. All rights reserved. + +using PineconeIntegrationTests.Support; +using VectorDataSpecificationTests.VectorSearch; +using Xunit; + +namespace PineconeIntegrationTests.VectorSearch; + +public class PineconeVectorSearchDistanceFunctionComplianceTests(PineconeFixture fixture) + : VectorSearchDistanceFunctionComplianceTests(fixture), IClassFixture +{ + public override Task CosineDistance() + => Assert.ThrowsAsync(base.CosineDistance); + + public override async Task CosineSimilarity() + { + await base.CosineSimilarity(); + await ArtificialDelayToWorkaroundEmulatorLimitations(); + } + + public override async Task DotProductSimilarity() + { + await base.DotProductSimilarity(); + await ArtificialDelayToWorkaroundEmulatorLimitations(); + } + + public override Task EuclideanDistance() + => Assert.ThrowsAsync(base.EuclideanDistance); + + public override async Task EuclideanSquaredDistance() + { + await base.EuclideanSquaredDistance(); + await ArtificialDelayToWorkaroundEmulatorLimitations(); + } + + public override Task Hamming() + => Assert.ThrowsAsync(base.Hamming); + + public override Task ManhattanDistance() + => Assert.ThrowsAsync(base.ManhattanDistance); + + public override Task NegativeDotProductSimilarity() + => Assert.ThrowsAsync(base.NegativeDotProductSimilarity); + + // The Pinecone emulator needs some extra time to spawn a new index service + // that uses a different distance function. + private static Task ArtificialDelayToWorkaroundEmulatorLimitations() + => Task.Delay(TimeSpan.FromSeconds(5)); +} diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/VectorSearch/VectorSearchDistanceFunctionComplianceTests.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/VectorSearch/VectorSearchDistanceFunctionComplianceTests.cs index 285c93c23e92..16c6a5f46c7b 100644 --- a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/VectorSearch/VectorSearchDistanceFunctionComplianceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/VectorSearch/VectorSearchDistanceFunctionComplianceTests.cs @@ -52,8 +52,14 @@ protected async Task SimpleSearch(string distanceFunction, double expectedExactM ReadOnlyMemory orthogonalVector = new([0f, -1f, -1f, 0f]); double[] scoreDictionary = [expectedExactMatchScore, expectedOppositeScore, expectedOrthogonalScore]; + double[] expectedScores = + [ + scoreDictionary[resultOrder[0]], + scoreDictionary[resultOrder[1]], + scoreDictionary[resultOrder[2]] + ]; - List records = + List insertedRecords = [ new() { @@ -76,10 +82,16 @@ protected async Task SimpleSearch(string distanceFunction, double expectedExactM Vector = orthogonalVector, } ]; + SearchRecord[] expectedRecords = + [ + insertedRecords[resultOrder[0]], + insertedRecords[resultOrder[1]], + insertedRecords[resultOrder[2]] + ]; // The record definition describes the distance function, // so we need a dedicated collection per test. - string uniqueCollectionName = Guid.NewGuid().ToString(); + string uniqueCollectionName = fixture.GetUniqueCollectionName(); var collection = fixture.TestStore.DefaultVectorStore.GetCollection( uniqueCollectionName, this.GetRecordDefinition(distanceFunction)); @@ -89,35 +101,55 @@ protected async Task SimpleSearch(string distanceFunction, double expectedExactM try { - await collection.UpsertBatchAsync(records).ToArrayAsync(); + await collection.UpsertBatchAsync(insertedRecords).ToArrayAsync(); var searchResult = await collection.VectorizedSearchAsync(baseVector); var results = await searchResult.Results.ToListAsync(); - VerifySearchResults(resultOrder, scoreDictionary, records, results, includeVectors: false); + VerifySearchResults(expectedRecords, expectedScores, results, includeVectors: false); searchResult = await collection.VectorizedSearchAsync(baseVector, new() { IncludeVectors = true }); results = await searchResult.Results.ToListAsync(); - VerifySearchResults(resultOrder, scoreDictionary, records, results, includeVectors: true); + VerifySearchResults(expectedRecords, expectedScores, results, includeVectors: true); + + for (int skip = 0; skip <= insertedRecords.Count; skip++) + { + for (int top = Math.Max(1, skip); top <= insertedRecords.Count; top++) + { + searchResult = await collection.VectorizedSearchAsync(baseVector, + new() + { + Skip = skip, + Top = top, + IncludeVectors = true + }); + results = await searchResult.Results.ToListAsync(); + + VerifySearchResults( + expectedRecords.Skip(skip).Take(top).ToArray(), + expectedScores.Skip(skip).Take(top).ToArray(), + results, includeVectors: true); + } + } } finally { await collection.DeleteCollectionAsync(); } - static void VerifySearchResults(int[] resultOrder, double[] scoreDictionary, List records, + static void VerifySearchResults(SearchRecord[] expectedRecords, double[] expectedScores, List> results, bool includeVectors) { - Assert.Equal(records.Count, results.Count); + Assert.Equal(expectedRecords.Length, results.Count); for (int i = 0; i < results.Count; i++) { - Assert.Equal(records[resultOrder[i]].Key, results[i].Record.Key); - Assert.Equal(records[resultOrder[i]].Int, results[i].Record.Int); - Assert.Equal(records[resultOrder[i]].String, results[i].Record.String); - Assert.Equal(Math.Round(scoreDictionary[resultOrder[i]], 2), Math.Round(results[i].Score!.Value, 2)); + Assert.Equal(expectedRecords[i].Key, results[i].Record.Key); + Assert.Equal(expectedRecords[i].Int, results[i].Record.Int); + Assert.Equal(expectedRecords[i].String, results[i].Record.String); + Assert.Equal(Math.Round(expectedScores[i], 2), Math.Round(results[i].Score!.Value, 2)); if (includeVectors) { - Assert.Equal(records[resultOrder[i]].Vector.ToArray(), results[i].Record.Vector.ToArray()); + Assert.Equal(expectedRecords[i].Vector.ToArray(), results[i].Record.Vector.ToArray()); } else {