diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props
index 4e70ab8174b9..62c5b7c38946 100644
--- a/dotnet/Directory.Packages.props
+++ b/dotnet/Directory.Packages.props
@@ -64,7 +64,7 @@
-
+
diff --git a/dotnet/SK-dotnet.sln b/dotnet/SK-dotnet.sln
index bad51cba9c8e..03ac72d9dee1 100644
--- a/dotnet/SK-dotnet.sln
+++ b/dotnet/SK-dotnet.sln
@@ -463,6 +463,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CosmosMongoDBIntegrationTes
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureAISearchIntegrationTests", "src\VectorDataIntegrationTests\AzureAISearchIntegrationTests\AzureAISearchIntegrationTests.csproj", "{06181F0F-A375-43AE-B45F-73CBCFC30C14}"
EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PineconeIntegrationTests", "src\VectorDataIntegrationTests\PineconeIntegrationTests\PineconeIntegrationTests.csproj", "{9D37CD08-620D-4AAC-9FEC-A8126AD8AB56}"
+EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Agents.AzureAI", "src\Agents\AzureAI\Agents.AzureAI.csproj", "{EA35F1B5-9148-4189-BE34-5E00AED56D65}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Plugins.AI", "src\Plugins\Plugins.AI\Plugins.AI.csproj", "{0C64EC81-8116-4388-87AD-BA14D4B59974}"
@@ -1149,6 +1151,12 @@ Global
{39EAB599-742F-417D-AF80-95F90376BB18}.Publish|Any CPU.Build.0 = Publish|Any CPU
{39EAB599-742F-417D-AF80-95F90376BB18}.Release|Any CPU.ActiveCfg = Release|Any CPU
{39EAB599-742F-417D-AF80-95F90376BB18}.Release|Any CPU.Build.0 = Release|Any CPU
+ {232E1153-6366-4175-A982-D66B30AAD610}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {232E1153-6366-4175-A982-D66B30AAD610}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {232E1153-6366-4175-A982-D66B30AAD610}.Publish|Any CPU.ActiveCfg = Debug|Any CPU
+ {232E1153-6366-4175-A982-D66B30AAD610}.Publish|Any CPU.Build.0 = Debug|Any CPU
+ {232E1153-6366-4175-A982-D66B30AAD610}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {232E1153-6366-4175-A982-D66B30AAD610}.Release|Any CPU.Build.0 = Release|Any CPU
{DAC54048-A39A-4739-8307-EA5A291F2EA0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{DAC54048-A39A-4739-8307-EA5A291F2EA0}.Debug|Any CPU.Build.0 = Debug|Any CPU
{DAC54048-A39A-4739-8307-EA5A291F2EA0}.Publish|Any CPU.ActiveCfg = Debug|Any CPU
@@ -1287,6 +1295,12 @@ Global
{06181F0F-A375-43AE-B45F-73CBCFC30C14}.Publish|Any CPU.Build.0 = Debug|Any CPU
{06181F0F-A375-43AE-B45F-73CBCFC30C14}.Release|Any CPU.ActiveCfg = Release|Any CPU
{06181F0F-A375-43AE-B45F-73CBCFC30C14}.Release|Any CPU.Build.0 = Release|Any CPU
+ {9D37CD08-620D-4AAC-9FEC-A8126AD8AB56}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {9D37CD08-620D-4AAC-9FEC-A8126AD8AB56}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {9D37CD08-620D-4AAC-9FEC-A8126AD8AB56}.Publish|Any CPU.ActiveCfg = Debug|Any CPU
+ {9D37CD08-620D-4AAC-9FEC-A8126AD8AB56}.Publish|Any CPU.Build.0 = Debug|Any CPU
+ {9D37CD08-620D-4AAC-9FEC-A8126AD8AB56}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {9D37CD08-620D-4AAC-9FEC-A8126AD8AB56}.Release|Any CPU.Build.0 = Release|Any CPU
{EA35F1B5-9148-4189-BE34-5E00AED56D65}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{EA35F1B5-9148-4189-BE34-5E00AED56D65}.Debug|Any CPU.Build.0 = Debug|Any CPU
{EA35F1B5-9148-4189-BE34-5E00AED56D65}.Publish|Any CPU.ActiveCfg = Publish|Any CPU
@@ -1537,6 +1551,7 @@ Global
{A0E65043-6B00-4836-850F-000A52238914} = {4F381919-F1BE-47D8-8558-3187ED04A84F}
{11DFBF14-6FBA-41F0-B7F3-A288952D6FDB} = {4F381919-F1BE-47D8-8558-3187ED04A84F}
{06181F0F-A375-43AE-B45F-73CBCFC30C14} = {4F381919-F1BE-47D8-8558-3187ED04A84F}
+ {9D37CD08-620D-4AAC-9FEC-A8126AD8AB56} = {4F381919-F1BE-47D8-8558-3187ED04A84F}
{EA35F1B5-9148-4189-BE34-5E00AED56D65} = {6823CD5E-2ABE-41EB-B865-F86EC13F0CF9}
{0C64EC81-8116-4388-87AD-BA14D4B59974} = {D6D598DF-C17C-46F4-B2B9-CDE82E2DE132}
{03ACF9DD-00C9-4F2B-80F1-537E2151AF5F} = {D6D598DF-C17C-46F4-B2B9-CDE82E2DE132}
diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj b/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj
index b2127f5131b0..6f73bf6e5b55 100644
--- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj
+++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj
@@ -18,8 +18,12 @@
Pinecone connector for Semantic Kernel plugins and semantic memory
+
+ $(DefineConstants);NON_PUBLISH
+
+
-
+
diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeGenericDataModelMapper.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeGenericDataModelMapper.cs
index 496a848ed394..df783a230498 100644
--- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeGenericDataModelMapper.cs
+++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeGenericDataModelMapper.cs
@@ -34,7 +34,7 @@ public PineconeGenericDataModelMapper(
///
public Vector MapFromDataToStorageModel(VectorStoreGenericDataModel dataModel)
{
- var metadata = new MetadataMap();
+ var metadata = new Metadata();
// Map data properties.
foreach (var dataProperty in this._propertyReader.DataProperties)
@@ -42,9 +42,9 @@ public Vector MapFromDataToStorageModel(VectorStoreGenericDataModel data
if (dataModel.Data.TryGetValue(dataProperty.DataModelPropertyName, out var propertyValue))
{
var propertyStorageName = this._propertyReader.GetStoragePropertyName(dataProperty.DataModelPropertyName);
- metadata[propertyStorageName] = propertyValue == null ?
- new MetadataValue() :
- PineconeVectorStoreRecordFieldMapping.ConvertToMetadataValue(propertyValue);
+ metadata[propertyStorageName] = propertyValue is not null
+ ? PineconeVectorStoreRecordFieldMapping.ConvertToMetadataValue(propertyValue)
+ : null;
}
}
@@ -62,8 +62,8 @@ public Vector MapFromDataToStorageModel(VectorStoreGenericDataModel data
// TODO: what about sparse values?
var result = new Vector
{
- Id = (string)dataModel.Key,
- Values = values.ToArray(),
+ Id = dataModel.Key,
+ Values = values,
Metadata = metadata,
SparseValues = null
};
@@ -80,7 +80,7 @@ public VectorStoreGenericDataModel MapFromStorageToDataModel(Vector stor
// Set Vector.
if (options?.IncludeVectors is true)
{
- dataModel.Vectors.Add(this._propertyReader.FirstVectorPropertyName!, new ReadOnlyMemory(storageModel.Values));
+ dataModel.Vectors.Add(this._propertyReader.FirstVectorPropertyName!, storageModel.Values);
}
// Set Data.
@@ -91,9 +91,10 @@ public VectorStoreGenericDataModel MapFromStorageToDataModel(Vector stor
var propertyStorageName = this._propertyReader.GetStoragePropertyName(dataProperty.DataModelPropertyName);
if (storageModel.Metadata.TryGetValue(propertyStorageName, out var propertyValue))
{
- dataModel.Data[dataProperty.DataModelPropertyName] = PineconeVectorStoreRecordFieldMapping.ConvertFromMetadataValueToNativeType(
- propertyValue,
- dataProperty.PropertyType);
+ dataModel.Data[dataProperty.DataModelPropertyName] =
+ propertyValue is not null
+ ? PineconeVectorStoreRecordFieldMapping.ConvertFromMetadataValueToNativeType(propertyValue, dataProperty.PropertyType)
+ : null;
}
}
}
diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs
index 4f79810e641b..a072ea6e7336 100644
--- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs
+++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs
@@ -4,7 +4,6 @@
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Threading;
-using Grpc.Core;
using Microsoft.Extensions.VectorData;
using Pinecone;
using Sdk = Pinecone;
@@ -20,7 +19,6 @@ namespace Microsoft.SemanticKernel.Connectors.Pinecone;
public class PineconeVectorStore : IVectorStore
{
private const string DatabaseName = "Pinecone";
- private const string ListCollectionsName = "ListCollections";
private readonly Sdk.PineconeClient _pineconeClient;
private readonly PineconeVectorStoreOptions _options;
@@ -63,24 +61,27 @@ public virtual IVectorStoreRecordCollection GetCollection
public virtual async IAsyncEnumerable ListCollectionNamesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)
{
- IndexDetails[] collections;
+ IndexList indexList;
try
{
- collections = await this._pineconeClient.ListIndexes(cancellationToken).ConfigureAwait(false);
+ indexList = await this._pineconeClient.ListIndexesAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
}
- catch (RpcException ex)
+ catch (PineconeApiException ex)
{
throw new VectorStoreOperationException("Call to vector store failed.", ex)
{
VectorStoreType = DatabaseName,
- OperationName = ListCollectionsName
+ OperationName = "ListCollections"
};
}
- foreach (var collection in collections)
+ if (indexList.Indexes is not null)
{
- yield return collection.Name;
+ foreach (var index in indexList.Indexes)
+ {
+ yield return index.Name;
+ }
}
}
}
diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionCreateMapping.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionCreateMapping.cs
deleted file mode 100644
index 5f8d6bf6137d..000000000000
--- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionCreateMapping.cs
+++ /dev/null
@@ -1,46 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System;
-using Microsoft.Extensions.VectorData;
-using Pinecone;
-
-namespace Microsoft.SemanticKernel.Connectors.Pinecone;
-
-///
-/// Contains mapping helpers to use when creating a Pinecone vector collection.
-///
-internal static class PineconeVectorStoreCollectionCreateMapping
-{
- ///
- /// Maps information stored in to a structure used by Pinecone SDK to create a serverless index.
- ///
- /// The property to map.
- /// The structure containing settings used to create a serverless index.
- /// Thrown if the property is missing information or has unsupported options specified.
- public static (uint Dimension, Metric Metric) MapServerlessIndex(VectorStoreRecordVectorProperty vectorProperty)
- {
- if (vectorProperty!.Dimensions is not > 0)
- {
- throw new InvalidOperationException($"Property {nameof(vectorProperty.Dimensions)} on {nameof(VectorStoreRecordVectorProperty)} '{vectorProperty.DataModelPropertyName}' must be set to a positive integer to create a collection.");
- }
-
- return (Dimension: (uint)vectorProperty.Dimensions, Metric: GetSDKMetricAlgorithm(vectorProperty));
- }
-
- ///
- /// Get the configured from the given .
- /// If none is configured, the default is .
- ///
- /// The vector property definition.
- /// The chosen .
- /// Thrown if a distance function is chosen that isn't supported by Pinecone.
- public static Metric GetSDKMetricAlgorithm(VectorStoreRecordVectorProperty vectorProperty)
- => vectorProperty.DistanceFunction switch
- {
- DistanceFunction.CosineSimilarity => Metric.Cosine,
- DistanceFunction.DotProductSimilarity => Metric.DotProduct,
- DistanceFunction.EuclideanSquaredDistance => Metric.Euclidean,
- null => Metric.Cosine,
- _ => throw new InvalidOperationException($"Distance function '{vectorProperty.DistanceFunction}' for {nameof(VectorStoreRecordVectorProperty)} '{vectorProperty.DataModelPropertyName}' is not supported by the Pinecone VectorStore.")
- };
-}
diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionSearchMapping.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionSearchMapping.cs
index 5b3d511c6b08..f5106bb251c1 100644
--- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionSearchMapping.cs
+++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionSearchMapping.cs
@@ -14,15 +14,15 @@ internal static class PineconeVectorStoreCollectionSearchMapping
{
#pragma warning disable CS0618 // FilterClause is obsolete
///
- /// Build a Pinecone from a set of filter clauses.
+ /// Build a Pinecone from a set of filter clauses.
///
- /// The filter clauses to build the Pinecone from.
+ /// The filter clauses to build the Pinecone from.
/// A mapping from property name to the name under which the property would be stored.
- /// The Pinecone .
+ /// The Pinecone .
/// Thrown for invalid property names, value types or filter clause types.
- public static MetadataMap BuildSearchFilter(IEnumerable? filterClauses, IReadOnlyDictionary storagePropertyNamesMap)
+ public static Metadata BuildSearchFilter(IEnumerable? filterClauses, IReadOnlyDictionary storagePropertyNamesMap)
{
- var metadataMap = new MetadataMap();
+ var metadataMap = new Metadata();
if (filterClauses is null)
{
@@ -46,7 +46,6 @@ public static MetadataMap BuildSearchFilter(IEnumerable? filterCla
bool boolValue => (MetadataValue)boolValue,
float floatValue => (MetadataValue)floatValue,
double doubleValue => (MetadataValue)doubleValue,
- decimal decimalValue => (MetadataValue)decimalValue,
_ => throw new InvalidOperationException($"Unsupported filter value type '{equalToFilterClause.Value.GetType().Name}'.")
};
diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs
index 6e44feda9334..8442482961a5 100644
--- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs
+++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs
@@ -6,10 +6,8 @@
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
-using Grpc.Core;
using Microsoft.Extensions.VectorData;
using Pinecone;
-using Pinecone.Grpc;
using Sdk = Pinecone;
namespace Microsoft.SemanticKernel.Connectors.Pinecone;
@@ -23,14 +21,6 @@ public class PineconeVectorStoreRecordCollection : IVectorStoreRecordCo
#pragma warning restore CA1711 // Identifiers should not have incorrect suffix
{
private const string DatabaseName = "Pinecone";
- private const string CreateCollectionName = "CreateCollection";
- private const string CollectionExistsName = "CollectionExists";
- private const string DeleteCollectionName = "DeleteCollection";
-
- private const string UpsertOperationName = "Upsert";
- private const string DeleteOperationName = "Delete";
- private const string GetOperationName = "Get";
- private const string QueryOperationName = "Query";
private static readonly VectorSearchOptions s_defaultVectorSearchOptions = new();
@@ -38,8 +28,7 @@ public class PineconeVectorStoreRecordCollection : IVectorStoreRecordCo
private readonly PineconeVectorStoreRecordCollectionOptions _options;
private readonly VectorStoreRecordPropertyReader _propertyReader;
private readonly IVectorStoreRecordMapper _mapper;
-
- private Sdk.Index? _index;
+ private IndexClient? _indexClient;
///
public string CollectionName { get; }
@@ -90,36 +79,39 @@ public PineconeVectorStoreRecordCollection(Sdk.PineconeClient pineconeClient, st
}
///
- public virtual async Task CollectionExistsAsync(CancellationToken cancellationToken = default)
- {
- var result = await this.RunOperationAsync(
- CollectionExistsName,
+ public virtual Task CollectionExistsAsync(CancellationToken cancellationToken = default)
+ => this.RunOperationAsync(
+ "CollectionExists",
async () =>
{
- var collections = await this._pineconeClient.ListIndexes(cancellationToken).ConfigureAwait(false);
+ var collections = await this._pineconeClient.ListIndexesAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
- return collections.Any(x => x.Name == this.CollectionName);
- }).ConfigureAwait(false);
-
- return result;
- }
+ return collections.Indexes?.Any(x => x.Name == this.CollectionName) is true;
+ });
///
- public virtual async Task CreateCollectionAsync(CancellationToken cancellationToken = default)
+ public virtual Task CreateCollectionAsync(CancellationToken cancellationToken = default)
{
// we already run through record property validation, so a single VectorStoreRecordVectorProperty is guaranteed.
var vectorProperty = this._propertyReader.VectorProperty!;
- var (dimension, metric) = PineconeVectorStoreCollectionCreateMapping.MapServerlessIndex(vectorProperty);
- await this.RunOperationAsync(
- CreateCollectionName,
- () => this._pineconeClient.CreateServerlessIndex(
- this.CollectionName,
- dimension,
- metric,
- this._options.ServerlessIndexCloud,
- this._options.ServerlessIndexRegion,
- cancellationToken)).ConfigureAwait(false);
+ CreateIndexRequest request = new()
+ {
+ Name = this.CollectionName,
+ Dimension = vectorProperty.Dimensions ?? throw new InvalidOperationException($"Property {nameof(vectorProperty.Dimensions)} on {nameof(VectorStoreRecordVectorProperty)} '{vectorProperty.DataModelPropertyName}' must be set to a positive integer to create a collection."),
+ Metric = MapDistanceFunction(vectorProperty),
+ Spec = new ServerlessIndexSpec
+ {
+ Serverless = new ServerlessSpec
+ {
+ Cloud = MapCloud(this._options.ServerlessIndexCloud),
+ Region = this._options.ServerlessIndexRegion,
+ }
+ },
+ };
+
+ return this.RunOperationAsync("CreateCollection",
+ () => this._pineconeClient.CreateIndexAsync(request, cancellationToken: cancellationToken));
}
///
@@ -127,24 +119,60 @@ public virtual async Task CreateCollectionIfNotExistsAsync(CancellationToken can
{
if (!await this.CollectionExistsAsync(cancellationToken).ConfigureAwait(false))
{
- await this.CreateCollectionAsync(cancellationToken).ConfigureAwait(false);
+ try
+ {
+ await this.CreateCollectionAsync(cancellationToken).ConfigureAwait(false);
+ }
+ catch (VectorStoreOperationException ex) when (ex.InnerException is PineconeApiException apiEx && apiEx.InnerException is ConflictError)
+ {
+ // If the collection got created in the meantime, we should ignore the exception.
+ }
}
}
///
public virtual Task DeleteCollectionAsync(CancellationToken cancellationToken = default)
=> this.RunOperationAsync(
- DeleteCollectionName,
- () => this._pineconeClient.DeleteIndex(this.CollectionName, cancellationToken));
+ "DeleteCollection",
+ async () =>
+ {
+ try
+ {
+ await this._pineconeClient.DeleteIndexAsync(this.CollectionName, cancellationToken: cancellationToken).ConfigureAwait(false);
+ }
+ catch (NotFoundError)
+ {
+ // If the collection does not exist, we should ignore the exception.
+ }
+ });
///
public virtual async Task GetAsync(string key, GetRecordOptions? options = null, CancellationToken cancellationToken = default)
{
Verify.NotNull(key);
- var records = await this.GetBatchAsync([key], options, cancellationToken).ToListAsync(cancellationToken).ConfigureAwait(false);
+ Sdk.FetchRequest request = new()
+ {
+ Namespace = this._options.IndexNamespace,
+ Ids = [key]
+ };
+
+ var response = await this.RunOperationAsync(
+ "Get",
+ () => this.GetIndexClient().FetchAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false);
- return records.FirstOrDefault();
+ var result = response.Vectors?.Values.FirstOrDefault();
+ if (result is null)
+ {
+ return default;
+ }
+
+ StorageToDataModelMapperOptions mapperOptions = new() { IncludeVectors = options?.IncludeVectors is true };
+ return VectorStoreErrorHandler.RunModelConversion(
+ DatabaseName,
+ this.CollectionName,
+ "Get",
+ () => this._mapper.MapFromStorageToDataModel(result, mapperOptions));
}
///
@@ -155,20 +183,32 @@ public virtual async IAsyncEnumerable GetBatchAsync(
{
Verify.NotNull(keys);
- var indexNamespace = this.GetIndexNamespace();
- var mapperOptions = new StorageToDataModelMapperOptions { IncludeVectors = options?.IncludeVectors ?? false };
-
- var index = await this.GetIndexAsync(this.CollectionName, cancellationToken).ConfigureAwait(false);
+ List keysList = keys.ToList();
+ if (keysList.Count == 0)
+ {
+ yield break;
+ }
- var results = await this.RunOperationAsync(
- GetOperationName,
- () => index.Fetch(keys, indexNamespace, cancellationToken)).ConfigureAwait(false);
+ Sdk.FetchRequest request = new()
+ {
+ Namespace = this._options.IndexNamespace,
+ Ids = keysList
+ };
+
+ var response = await this.RunOperationAsync(
+ "GetBatch",
+ () => this.GetIndexClient().FetchAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false);
+ if (response.Vectors is null || response.Vectors.Count == 0)
+ {
+ yield break;
+ }
+ StorageToDataModelMapperOptions mapperOptions = new() { IncludeVectors = options?.IncludeVectors is true };
var records = VectorStoreErrorHandler.RunModelConversion(
DatabaseName,
this.CollectionName,
- GetOperationName,
- () => results.Values.Select(x => this._mapper.MapFromStorageToDataModel(x, mapperOptions)));
+ "GetBatch",
+ () => response.Vectors.Values.Select(x => this._mapper.MapFromStorageToDataModel(x, mapperOptions)));
foreach (var record in records)
{
@@ -181,21 +221,37 @@ public virtual Task DeleteAsync(string key, CancellationToken cancellationToken
{
Verify.NotNullOrWhiteSpace(key);
- return this.DeleteBatchAsync([key], cancellationToken);
+ Sdk.DeleteRequest request = new()
+ {
+ Namespace = this._options.IndexNamespace,
+ Ids = [key]
+ };
+
+ return this.RunOperationAsync(
+ "Delete",
+ () => this.GetIndexClient().DeleteAsync(request, cancellationToken: cancellationToken));
}
///
- public virtual async Task DeleteBatchAsync(IEnumerable keys, CancellationToken cancellationToken = default)
+ public virtual Task DeleteBatchAsync(IEnumerable keys, CancellationToken cancellationToken = default)
{
Verify.NotNull(keys);
- var indexNamespace = this.GetIndexNamespace();
+ List keysList = keys.ToList();
+ if (keysList.Count == 0)
+ {
+ return Task.CompletedTask;
+ }
- var index = await this.GetIndexAsync(this.CollectionName, cancellationToken).ConfigureAwait(false);
+ Sdk.DeleteRequest request = new()
+ {
+ Namespace = this._options.IndexNamespace,
+ Ids = keysList
+ };
- await this.RunOperationAsync(
- DeleteOperationName,
- () => index.Delete(keys, indexNamespace, cancellationToken)).ConfigureAwait(false);
+ return this.RunOperationAsync(
+ "DeleteBatch",
+ () => this.GetIndexClient().DeleteAsync(request, cancellationToken: cancellationToken));
}
///
@@ -203,19 +259,21 @@ public virtual async Task UpsertAsync(TRecord record, CancellationToken
{
Verify.NotNull(record);
- var indexNamespace = this.GetIndexNamespace();
-
- var index = await this.GetIndexAsync(this.CollectionName, cancellationToken).ConfigureAwait(false);
-
var vector = VectorStoreErrorHandler.RunModelConversion(
DatabaseName,
this.CollectionName,
- UpsertOperationName,
+ "Upsert",
() => this._mapper.MapFromDataToStorageModel(record));
+ Sdk.UpsertRequest request = new()
+ {
+ Namespace = this._options.IndexNamespace,
+ Vectors = [vector],
+ };
+
await this.RunOperationAsync(
- UpsertOperationName,
- () => index.Upsert([vector], indexNamespace, cancellationToken)).ConfigureAwait(false);
+ "Upsert",
+ () => this.GetIndexClient().UpsertAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false);
return vector.Id;
}
@@ -225,19 +283,26 @@ public virtual async IAsyncEnumerable UpsertBatchAsync(IEnumerable records.Select(this._mapper.MapFromDataToStorageModel).ToList());
+ if (vectors.Count == 0)
+ {
+ yield break;
+ }
+
+ Sdk.UpsertRequest request = new()
+ {
+ Namespace = this._options.IndexNamespace,
+ Vectors = vectors,
+ };
+
await this.RunOperationAsync(
- UpsertOperationName,
- () => index.Upsert(vectors, indexNamespace, cancellationToken)).ConfigureAwait(false);
+ "UpsertBatch",
+ () => this.GetIndexClient().UpsertAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false);
foreach (var vector in vectors)
{
@@ -256,61 +321,52 @@ public virtual async Task> VectorizedSearchAsync).FullName}");
}
- // Resolve options and build filter clause.
- var internalOptions = options ?? s_defaultVectorSearchOptions;
- var mapperOptions = new StorageToDataModelMapperOptions { IncludeVectors = options?.IncludeVectors ?? false };
+ options ??= s_defaultVectorSearchOptions;
#pragma warning disable CS0618 // FilterClause is obsolete
var filter = PineconeVectorStoreCollectionSearchMapping.BuildSearchFilter(
- internalOptions.OldFilter?.FilterClauses,
+ options.OldFilter?.FilterClauses,
this._propertyReader.StoragePropertyNamesMap);
#pragma warning restore CS0618
- // Get the current index.
- var indexNamespace = this.GetIndexNamespace();
- var index = await this.GetIndexAsync(this.CollectionName, cancellationToken).ConfigureAwait(false);
-
- // Search.
- var results = await this.RunOperationAsync(
- QueryOperationName,
- () => index.Query(
- floatVector.ToArray(),
- (uint)(internalOptions.Skip + internalOptions.Top),
- filter,
- sparseValues: null,
- indexNamespace,
- internalOptions.IncludeVectors,
- includeMetadata: true,
- cancellationToken)).ConfigureAwait(false);
-
- // Skip the required results for paging.
- var skippedResults = results.Skip(internalOptions.Skip);
-
- // Map the results.
+ Sdk.QueryRequest request = new()
+ {
+ TopK = (uint)(options.Top + options.Skip),
+ Namespace = this._options.IndexNamespace,
+ IncludeValues = options.IncludeVectors,
+ IncludeMetadata = true,
+ Vector = floatVector,
+ Filter = filter,
+ };
+
+ Sdk.QueryResponse response = await this.RunOperationAsync(
+ "Query",
+ () => this.GetIndexClient().QueryAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false);
+
+ if (response.Matches is null)
+ {
+ return new VectorSearchResults(Array.Empty>().ToAsyncEnumerable());
+ }
+
+ // Pinecone does not provide a way to skip results, so we need to do it manually.
+ var skippedResults = response.Matches
+ .Skip(options.Skip);
+
+ StorageToDataModelMapperOptions mapperOptions = new() { IncludeVectors = options.IncludeVectors is true };
var records = VectorStoreErrorHandler.RunModelConversion(
DatabaseName,
this.CollectionName,
- QueryOperationName,
- () =>
+ "Query",
+ () => skippedResults.Select(x => new VectorSearchResult(this._mapper.MapFromStorageToDataModel(new Sdk.Vector()
{
- // First convert to Vector objects, since the
- // mapper requires these as input.
- var vectorResults = skippedResults.Select(x => (
- Vector: new Vector()
- {
- Id = x.Id,
- Values = x.Values ?? Array.Empty(),
- Metadata = x.Metadata,
- SparseValues = x.SparseValues
- },
- x.Score));
-
- return vectorResults.Select(x => new VectorSearchResult(
- this._mapper.MapFromStorageToDataModel(x.Vector, mapperOptions),
- x.Score));
- });
-
- return new VectorSearchResults(records.ToAsyncEnumerable());
+ Id = x.Id,
+ Values = x.Values ?? Array.Empty(),
+ Metadata = x.Metadata,
+ SparseValues = x.SparseValues
+ }, mapperOptions), x.Score)))
+ .ToAsyncEnumerable();
+
+ return new(records);
}
private async Task RunOperationAsync(string operationName, Func> operation)
@@ -319,7 +375,7 @@ private async Task RunOperationAsync(string operationName, Func> o
{
return await operation.Invoke().ConfigureAwait(false);
}
- catch (RpcException ex)
+ catch (PineconeApiException ex)
{
throw new VectorStoreOperationException("Call to vector store failed.", ex)
{
@@ -336,7 +392,7 @@ private async Task RunOperationAsync(string operationName, Func operation)
{
await operation.Invoke().ConfigureAwait(false);
}
- catch (RpcException ex)
+ catch (PineconeApiException ex)
{
throw new VectorStoreOperationException("Call to vector store failed.", ex)
{
@@ -347,13 +403,44 @@ private async Task RunOperationAsync(string operationName, Func operation)
}
}
- private async Task> GetIndexAsync(string indexName, CancellationToken cancellationToken)
+ private IndexClient GetIndexClient()
{
- this._index ??= await this._pineconeClient.GetIndex(indexName, cancellationToken).ConfigureAwait(false);
+ if (this._indexClient is null)
+ {
+#if NON_PUBLISH
+ // When "host" is not provided for PineconeClient.Index,
+ // it will try to get the host from the Pinecone service.
+ // In the cloud environment it's fine, but with the local emulator
+ // it reports the address of the REST endpoint rather than the gRPC one.
+ // To work around this, we set the environment variable to the gRPC endpoint.
+ // But only for the non-publish build, so this logic is not used in production.
+ string? hostName = Environment.GetEnvironmentVariable("PINECONE_GRPC_ENDPOINT");
+#else
+ string? hostName = null;
+#endif
+
+ this._indexClient = this._pineconeClient.Index(name: this.CollectionName, hostName);
+ }
- return this._index;
+ return this._indexClient;
}
- private string? GetIndexNamespace()
- => this._options.IndexNamespace;
+ private static ServerlessSpecCloud MapCloud(string serverlessIndexCloud)
+ => serverlessIndexCloud switch
+ {
+ "aws" => ServerlessSpecCloud.Aws,
+ "azure" => ServerlessSpecCloud.Azure,
+ "gcp" => ServerlessSpecCloud.Gcp,
+ _ => throw new ArgumentException($"Invalid serverless index cloud: {serverlessIndexCloud}.", nameof(serverlessIndexCloud))
+ };
+
+ private static CreateIndexRequestMetric MapDistanceFunction(VectorStoreRecordVectorProperty vectorProperty)
+ => vectorProperty.DistanceFunction switch
+ {
+ DistanceFunction.CosineSimilarity => CreateIndexRequestMetric.Cosine,
+ DistanceFunction.DotProductSimilarity => CreateIndexRequestMetric.Dotproduct,
+ DistanceFunction.EuclideanSquaredDistance => CreateIndexRequestMetric.Euclidean,
+ null => CreateIndexRequestMetric.Cosine,
+ _ => throw new NotSupportedException($"Distance function '{vectorProperty.DistanceFunction}' is not supported.")
+ };
}
diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordFieldMapping.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordFieldMapping.cs
index 6acbec24d72c..9573740f8580 100644
--- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordFieldMapping.cs
+++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordFieldMapping.cs
@@ -48,7 +48,7 @@ internal static class PineconeVectorStoreRecordFieldMapping
];
public static object? ConvertFromMetadataValueToNativeType(MetadataValue metadataValue, Type targetType)
- => metadataValue.Inner switch
+ => metadataValue.Value switch
{
null => null,
bool boolValue => boolValue,
@@ -59,26 +59,28 @@ internal static class PineconeVectorStoreRecordFieldMapping
long longValue => ConvertToNumericValue(longValue, targetType),
float floatValue => ConvertToNumericValue(floatValue, targetType),
double doubleValue => ConvertToNumericValue(doubleValue, targetType),
- decimal decimalValue => ConvertToNumericValue(decimalValue, targetType),
MetadataValue[] array => VectorStoreRecordMapping.CreateEnumerable(array.Select(x => ConvertFromMetadataValueToNativeType(x, VectorStoreRecordPropertyVerification.GetCollectionElementType(targetType))), targetType),
List list => VectorStoreRecordMapping.CreateEnumerable(list.Select(x => ConvertFromMetadataValueToNativeType(x, VectorStoreRecordPropertyVerification.GetCollectionElementType(targetType))), targetType),
- _ => throw new VectorStoreRecordMappingException($"Unsupported metadata type: '{metadataValue.Inner?.GetType().FullName}'."),
+ _ => throw new VectorStoreRecordMappingException($"Unsupported metadata type: '{metadataValue.Value?.GetType().FullName}'."),
};
- // TODO: take advantage of MetadataValue.TryCreate once we upgrade the version of Pinecone.NET
public static MetadataValue ConvertToMetadataValue(object? sourceValue)
=> sourceValue switch
{
bool boolValue => boolValue,
+ bool[] bools => bools,
+ List bools => bools,
string stringValue => stringValue,
+ string[] stringArray => stringArray,
+ List stringList => stringList,
+ double doubleValue => doubleValue,
+ double[] doubles => doubles,
+ List doubles => doubles,
+ // Other numeric types are simply cast into double in implicit way.
+ // We could consider supporting arrays of these types.
int intValue => intValue,
long longValue => longValue,
float floatValue => floatValue,
- double doubleValue => doubleValue,
- decimal decimalValue => decimalValue,
- string[] stringArray => stringArray,
- List stringList => stringList,
- IEnumerable stringEnumerable => stringEnumerable.ToArray(),
_ => throw new VectorStoreRecordMappingException($"Unsupported source value type '{sourceValue?.GetType().FullName}'.")
};
diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordMapper.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordMapper.cs
index 501937eaf50d..1163c1a66bea 100644
--- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordMapper.cs
+++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordMapper.cs
@@ -40,7 +40,7 @@ public Vector MapFromDataToStorageModel(TRecord dataModel)
throw new VectorStoreRecordMappingException($"Key property {this._propertyReader.KeyPropertyName} on provided record of type {typeof(TRecord).FullName} may not be null.");
}
- var metadata = new MetadataMap();
+ var metadata = new Metadata();
foreach (var dataPropertyInfo in this._propertyReader.DataPropertiesInfo)
{
var propertyName = this._propertyReader.GetStoragePropertyName(dataPropertyInfo.Name);
@@ -61,7 +61,7 @@ public Vector MapFromDataToStorageModel(TRecord dataModel)
var result = new Vector
{
Id = (string)keyObject,
- Values = values.ToArray(),
+ Values = values,
Metadata = metadata,
SparseValues = null
};
@@ -83,7 +83,7 @@ public TRecord MapFromStorageToDataModel(Vector storageModel, StorageToDataModel
{
this._propertyReader.FirstVectorPropertyInfo!.SetValue(
outputRecord,
- new ReadOnlyMemory(storageModel.Values));
+ storageModel.Values);
}
// Set Data.
@@ -94,7 +94,7 @@ public TRecord MapFromStorageToDataModel(Vector storageModel, StorageToDataModel
this._propertyReader.DataPropertiesInfo,
this._propertyReader.StoragePropertyNamesMap,
storageModel.Metadata,
- PineconeVectorStoreRecordFieldMapping.ConvertFromMetadataValueToNativeType);
+ PineconeVectorStoreRecordFieldMapping.ConvertFromMetadataValueToNativeType!);
}
return outputRecord;
diff --git a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs
index 14a8e56222d9..52f6091f129e 100644
--- a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs
+++ b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs
@@ -177,9 +177,16 @@ public virtual async Task CreateCollectionIfNotExistsAsync(CancellationToken can
}
///
- public virtual Task DeleteCollectionAsync(CancellationToken cancellationToken = default)
+ public virtual async Task DeleteCollectionAsync(CancellationToken cancellationToken = default)
{
- return this.RunOperationAsync("FT.DROPINDEX", () => this._database.FT().DropIndexAsync(this._collectionName));
+ try
+ {
+ await this.RunOperationAsync("FT.DROPINDEX", () => this._database.FT().DropIndexAsync(this._collectionName)).ConfigureAwait(false);
+ }
+ catch (VectorStoreOperationException ex) when (ex.InnerException is RedisServerException innerEx && innerEx.Message.Contains("Unknown Index name"))
+ {
+ // Collection does not exist.
+ }
}
///
diff --git a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeGenericDataModelMapperTests.cs b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeGenericDataModelMapperTests.cs
index 9a41450d9649..0a96bb41cc3b 100644
--- a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeGenericDataModelMapperTests.cs
+++ b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeGenericDataModelMapperTests.cs
@@ -77,20 +77,20 @@ public void MapFromDataToStorageModelMapsAllSupportedTypes()
// Assert
Assert.Equal(TestKeyString, storageModel.Id);
- Assert.Equal("string", (string?)storageModel.Metadata!["StringDataProp"].Inner);
+ Assert.Equal("string", (string?)storageModel.Metadata!["StringDataProp"]!.Value);
// MetadataValue converts all numeric types to double.
- Assert.Equal(1, (double?)storageModel.Metadata["IntDataProp"].Inner);
- Assert.Equal(2, (double?)storageModel.Metadata["NullableIntDataProp"].Inner);
- Assert.Equal(3L, (double?)storageModel.Metadata["LongDataProp"].Inner);
- Assert.Equal(4L, (double?)storageModel.Metadata["NullableLongDataProp"].Inner);
- Assert.Equal(5.0f, (double?)storageModel.Metadata["FloatDataProp"].Inner);
- Assert.Equal(6.0f, (double?)storageModel.Metadata["NullableFloatDataProp"].Inner);
- Assert.Equal(7.0, (double?)storageModel.Metadata["DoubleDataProp"].Inner);
- Assert.Equal(8.0, (double?)storageModel.Metadata["NullableDoubleDataProp"].Inner);
- Assert.Equal(true, (bool?)storageModel.Metadata["BoolDataProp"].Inner);
- Assert.Equal(false, (bool?)storageModel.Metadata["NullableBoolDataProp"].Inner);
- Assert.Equal(s_taglist, ((IEnumerable?)(storageModel.Metadata["TagListDataProp"].Inner!))
- .Select(x => x.Inner as string)
+ Assert.Equal(1, (double?)storageModel.Metadata["IntDataProp"]!.Value);
+ Assert.Equal(2, (double?)storageModel.Metadata["NullableIntDataProp"]!.Value);
+ Assert.Equal(3L, (double?)storageModel.Metadata["LongDataProp"]!.Value);
+ Assert.Equal(4L, (double?)storageModel.Metadata["NullableLongDataProp"]!.Value);
+ Assert.Equal(5.0f, (double?)storageModel.Metadata["FloatDataProp"]!.Value);
+ Assert.Equal(6.0f, (double?)storageModel.Metadata["NullableFloatDataProp"]!.Value);
+ Assert.Equal(7.0, (double?)storageModel.Metadata["DoubleDataProp"]!.Value);
+ Assert.Equal(8.0, (double?)storageModel.Metadata["NullableDoubleDataProp"]!.Value);
+ Assert.Equal(true, (bool?)storageModel.Metadata["BoolDataProp"]!.Value);
+ Assert.Equal(false, (bool?)storageModel.Metadata["NullableBoolDataProp"]!.Value);
+ Assert.Equal(s_taglist, ((IEnumerable?)(storageModel.Metadata["TagListDataProp"]!.Value!))
+ .Select(x => x.Value as string)
.ToArray());
Assert.Equal(s_vector, storageModel.Values);
}
@@ -136,9 +136,9 @@ public void MapFromDataToStorageModelMapsNullValues()
// Assert
Assert.Equal(TestKeyString, storageModel.Id);
- Assert.True(storageModel.Metadata!["StringDataProp"].Inner == null);
- Assert.True(storageModel.Metadata["NullableIntDataProp"].Inner == null);
- Assert.True(storageModel.Metadata["NullableTagListDataProp"].Inner == null);
+ Assert.Null(storageModel.Metadata!["StringDataProp"]);
+ Assert.Null(storageModel.Metadata["NullableIntDataProp"]);
+ Assert.Null(storageModel.Metadata["NullableTagListDataProp"]);
}
[Fact]
@@ -153,7 +153,7 @@ public void MapFromStorageToDataModelMapsAllSupportedTypes()
var storageModel = new Vector()
{
Id = TestKeyString,
- Metadata = new MetadataMap()
+ Metadata = new Metadata()
{
["StringDataProp"] = (MetadataValue)"string",
["IntDataProp"] = (MetadataValue)1,
@@ -168,7 +168,7 @@ public void MapFromStorageToDataModelMapsAllSupportedTypes()
["NullableBoolDataProp"] = (MetadataValue)false,
["TagListDataProp"] = (MetadataValue)new MetadataValue[] { "tag1", "tag2" }
},
- Values = [1.0f, 2.0f, 3.0f]
+ Values = new float[] { 1.0f, 2.0f, 3.0f }
};
// Act
@@ -210,13 +210,13 @@ public void MapFromStorageToDataModelMapsNullValues()
var storageModel = new Vector()
{
Id = TestKeyString,
- Metadata = new MetadataMap()
+ Metadata = new Metadata()
{
- ["StringDataProp"] = new MetadataValue(),
- ["NullableIntDataProp"] = new MetadataValue(),
- ["NullableTagListDataProp"] = new MetadataValue(),
+ ["StringDataProp"] = null,
+ ["NullableIntDataProp"] = null,
+ ["NullableTagListDataProp"] = null,
},
- Values = [1.0f, 2.0f, 3.0f]
+ Values = new float[] { 1.0f, 2.0f, 3.0f }
};
var reader = new VectorStoreRecordPropertyReader(
@@ -327,7 +327,7 @@ public void MapFromStorageToDataModelSkipsMissingProperties()
var storageModel = new Vector()
{
Id = TestKeyString,
- Values = [1.0f, 2.0f, 3.0f]
+ Values = new float[] { 1.0f, 2.0f, 3.0f }
};
// Act
diff --git a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeKernelBuilderExtensionsTests.cs b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeKernelBuilderExtensionsTests.cs
index 6777470e537d..21f7b6649da5 100644
--- a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeKernelBuilderExtensionsTests.cs
+++ b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeKernelBuilderExtensionsTests.cs
@@ -26,7 +26,7 @@ public PineconeKernelBuilderExtensionsTests()
public void AddVectorStoreRegistersClass()
{
// Arrange.
- using var client = new Sdk.PineconeClient("fake api key");
+ var client = new Sdk.PineconeClient("fake api key");
this._kernelBuilder.Services.AddSingleton(client);
// Act.
@@ -50,7 +50,7 @@ public void AddVectorStoreWithApiKeyRegistersClass()
public void AddVectorStoreRecordCollectionRegistersClass()
{
// Arrange.
- using var client = new Sdk.PineconeClient("fake api key");
+ var client = new Sdk.PineconeClient("fake api key");
this._kernelBuilder.Services.AddSingleton(client);
// Act.
diff --git a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeServiceCollectionExtensionsTests.cs b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeServiceCollectionExtensionsTests.cs
index 084222d37bda..736cc3e3839d 100644
--- a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeServiceCollectionExtensionsTests.cs
+++ b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeServiceCollectionExtensionsTests.cs
@@ -26,7 +26,7 @@ public PineconeServiceCollectionExtensionsTests()
public void AddVectorStoreRegistersClass()
{
// Arrange.
- using var client = new Sdk.PineconeClient("fake api key");
+ var client = new Sdk.PineconeClient("fake api key");
this._serviceCollection.AddSingleton(client);
// Act.
@@ -49,7 +49,7 @@ public void AddVectorStoreWithApiKeyRegistersClass()
public void AddVectorStoreRecordCollectionRegistersClass()
{
// Arrange.
- using var client = new Sdk.PineconeClient("fake api key");
+ var client = new Sdk.PineconeClient("fake api key");
this._serviceCollection.AddSingleton(client);
// Act.
diff --git a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeVectorStoreRecordCollectionTests.cs b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeVectorStoreRecordCollectionTests.cs
index 85ed14f7a468..0dc2620140f3 100644
--- a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeVectorStoreRecordCollectionTests.cs
+++ b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeVectorStoreRecordCollectionTests.cs
@@ -35,7 +35,7 @@ public void CanCreateCollectionWithMismatchedDefinitionAndType()
new VectorStoreRecordVectorProperty("Embedding", typeof(ReadOnlyMemory)) { Dimensions = 4 },
}
};
- using var pineconeClient = new Sdk.PineconeClient("fake api key");
+ var pineconeClient = new Sdk.PineconeClient("fake api key");
// Act.
var sut = new PineconeVectorStoreRecordCollection(
diff --git a/dotnet/src/Experimental/Process.IntegrationTestRunner.Dapr/Process.IntegrationTestRunner.Dapr.csproj b/dotnet/src/Experimental/Process.IntegrationTestRunner.Dapr/Process.IntegrationTestRunner.Dapr.csproj
index ba07a9a9dcad..2d35183b3648 100644
--- a/dotnet/src/Experimental/Process.IntegrationTestRunner.Dapr/Process.IntegrationTestRunner.Dapr.csproj
+++ b/dotnet/src/Experimental/Process.IntegrationTestRunner.Dapr/Process.IntegrationTestRunner.Dapr.csproj
@@ -32,7 +32,6 @@
-
diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/CommonPineconeVectorStoreRecordCollectionTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/CommonPineconeVectorStoreRecordCollectionTests.cs
deleted file mode 100644
index 21964a61e3d0..000000000000
--- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/CommonPineconeVectorStoreRecordCollectionTests.cs
+++ /dev/null
@@ -1,42 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using Microsoft.Extensions.VectorData;
-using Microsoft.SemanticKernel.Connectors.Pinecone;
-using SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone.Xunit;
-using Xunit;
-
-namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone;
-
-///
-/// Inherits common integration tests that should pass for any .
-///
-/// Pinecone setup and teardown.
-[Collection("PineconeVectorStoreTests")]
-[PineconeApiKeySetCondition]
-public class CommonPineconeVectorStoreRecordCollectionTests(PineconeVectorStoreFixture fixture) : BaseVectorStoreRecordCollectionTests, IClassFixture
-{
- protected override string Key1 => "1";
- protected override string Key2 => "2";
- protected override string Key3 => "3";
- protected override string Key4 => "4";
-
- protected override int DelayAfterIndexCreateInMilliseconds => 2000;
-
- protected override int DelayAfterUploadInMilliseconds => 15000;
-
- [SuppressMessage("Globalization", "CA1308:Normalize strings to uppercase", Justification = "Pinecone collection names should be lower case.")]
- protected override IVectorStoreRecordCollection GetTargetRecordCollection(string recordCollectionName, VectorStoreRecordDefinition? vectorStoreRecordDefinition)
- {
- return new PineconeVectorStoreRecordCollection(fixture.Client, recordCollectionName.ToLowerInvariant(), new()
- {
- VectorStoreRecordDefinition = vectorStoreRecordDefinition
- });
- }
-
- protected override HashSet GetSupportedDistanceFunctions()
- {
- return [DistanceFunction.CosineSimilarity, DistanceFunction.DotProductSimilarity, DistanceFunction.EuclideanSquaredDistance];
- }
-}
diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeAllTypes.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeAllTypes.cs
deleted file mode 100644
index 7067781987bc..000000000000
--- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeAllTypes.cs
+++ /dev/null
@@ -1,64 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System;
-using System.Collections.Generic;
-using Microsoft.Extensions.VectorData;
-
-namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone;
-
-#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
-public record PineconeAllTypes()
-{
- [VectorStoreRecordKey]
- public string Id { get; init; }
-
- [VectorStoreRecordData]
- public bool BoolProperty { get; set; }
- [VectorStoreRecordData]
- public bool? NullableBoolProperty { get; set; }
- [VectorStoreRecordData]
- public string StringProperty { get; set; }
- [VectorStoreRecordData]
- public string? NullableStringProperty { get; set; }
- [VectorStoreRecordData]
- public int IntProperty { get; set; }
- [VectorStoreRecordData]
- public int? NullableIntProperty { get; set; }
- [VectorStoreRecordData]
- public long LongProperty { get; set; }
- [VectorStoreRecordData]
- public long? NullableLongProperty { get; set; }
- [VectorStoreRecordData]
- public float FloatProperty { get; set; }
- [VectorStoreRecordData]
- public float? NullableFloatProperty { get; set; }
- [VectorStoreRecordData]
- public double DoubleProperty { get; set; }
- [VectorStoreRecordData]
- public double? NullableDoubleProperty { get; set; }
- [VectorStoreRecordData]
- public decimal DecimalProperty { get; set; }
- [VectorStoreRecordData]
- public decimal? NullableDecimalProperty { get; set; }
-
-#pragma warning disable CA1819 // Properties should not return arrays
- [VectorStoreRecordData]
- public string[] StringArray { get; set; }
- [VectorStoreRecordData]
- public string[]? NullableStringArray { get; set; }
-#pragma warning restore CA1819 // Properties should not return arrays
-
- [VectorStoreRecordData]
- public List StringList { get; set; }
- [VectorStoreRecordData]
- public List? NullableStringList { get; set; }
-
- [VectorStoreRecordData]
- public IReadOnlyCollection Collection { get; set; }
- [VectorStoreRecordData]
- public IEnumerable Enumerable { get; set; }
-
- [VectorStoreRecordVector(Dimensions: 8, DistanceFunction: DistanceFunction.DotProductSimilarity)]
- public ReadOnlyMemory? Embedding { get; set; }
-}
-#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeHotel.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeHotel.cs
deleted file mode 100644
index 54185830d5c0..000000000000
--- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeHotel.cs
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System;
-using System.Collections.Generic;
-using System.Text.Json.Serialization;
-using Microsoft.Extensions.VectorData;
-
-namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone;
-
-#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
-public record PineconeHotel()
-
-{
- [VectorStoreRecordKey]
- public string HotelId { get; init; }
-
- [VectorStoreRecordData]
- public string HotelName { get; set; }
-
- [JsonPropertyName("code_of_the_hotel")]
- [VectorStoreRecordData]
- public int HotelCode { get; set; }
-
- [VectorStoreRecordData]
- public float HotelRating { get; set; }
-
- [JsonPropertyName("json_parking")]
- [VectorStoreRecordData(StoragePropertyName = "parking_is_included")]
- public bool ParkingIncluded { get; set; }
-
- [VectorStoreRecordData]
- public List Tags { get; set; } = [];
-
- [VectorStoreRecordData]
- public string Description { get; set; }
-
- [VectorStoreRecordVector(Dimensions: 8, DistanceFunction: DistanceFunction.DotProductSimilarity)]
- public ReadOnlyMemory DescriptionEmbedding { get; set; }
-}
-#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeUserSecretsExtensions.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeUserSecretsExtensions.cs
deleted file mode 100644
index 1644b7427e99..000000000000
--- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeUserSecretsExtensions.cs
+++ /dev/null
@@ -1,37 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System.Collections.Generic;
-using System.IO;
-using System.Reflection;
-using System.Text.Json;
-using Microsoft.Extensions.Configuration.UserSecrets;
-
-namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone;
-public static class PineconeUserSecretsExtensions
-{
- public const string PineconeApiKeyUserSecretEntry = "PineconeApiKey";
-
- public static string ReadPineconeApiKey()
- => JsonSerializer.Deserialize>(
- File.ReadAllText(PathHelper.GetSecretsPathFromSecretsId(
- typeof(PineconeUserSecretsExtensions).Assembly.GetCustomAttribute()!
- .UserSecretsId)))![PineconeApiKeyUserSecretEntry].Trim();
-
- public static bool ContainsPineconeApiKey()
- {
- var userSecretsIdAttribute = typeof(PineconeUserSecretsExtensions).Assembly.GetCustomAttribute();
- if (userSecretsIdAttribute == null)
- {
- return false;
- }
-
- var path = PathHelper.GetSecretsPathFromSecretsId(userSecretsIdAttribute.UserSecretsId);
- if (!File.Exists(path))
- {
- return false;
- }
-
- return JsonSerializer.Deserialize>(
- File.ReadAllText(path))!.ContainsKey(PineconeApiKeyUserSecretEntry);
- }
-}
diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreFixture.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreFixture.cs
deleted file mode 100644
index c0c002f22ba0..000000000000
--- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreFixture.cs
+++ /dev/null
@@ -1,350 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Net.Http;
-using System.Text.RegularExpressions;
-using System.Threading.Tasks;
-using Microsoft.Extensions.VectorData;
-using Microsoft.SemanticKernel.Connectors.Pinecone;
-using Pinecone.Grpc;
-using Xunit;
-using Sdk = Pinecone;
-
-namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone;
-
-public class PineconeVectorStoreFixture : IAsyncLifetime
-{
- private const int MaxAttemptCount = 100;
- private const int DelayInterval = 300;
-
- public string IndexName { get; } = "sk-index"
-#pragma warning disable CA1308 // Normalize strings to uppercase
- + new Regex("[^a-zA-Z0-9]", RegexOptions.None, matchTimeout: new TimeSpan(0, 0, 10)).Replace(Environment.MachineName.ToLowerInvariant(), "");
-#pragma warning restore CA1308 // Normalize strings to uppercase
-
- public Sdk.PineconeClient Client { get; private set; } = null!;
- public PineconeVectorStore VectorStore { get; private set; } = null!;
- public PineconeVectorStoreRecordCollection HotelRecordCollection { get; set; } = null!;
- public PineconeVectorStoreRecordCollection AllTypesRecordCollection { get; set; } = null!;
- public PineconeVectorStoreRecordCollection HotelRecordCollectionWithCustomNamespace { get; set; } = null!;
- public IVectorStoreRecordCollection HotelRecordCollectionFromVectorStore { get; set; } = null!;
- public IVectorStoreRecordCollection> HotelRecordCollectionWithGenericDataModel { get; set; } = null!;
-
- public virtual Sdk.Index Index { get; set; } = null!;
-
- public virtual async Task InitializeAsync()
- {
- this.Client = new Sdk.PineconeClient(PineconeUserSecretsExtensions.ReadPineconeApiKey());
- this.VectorStore = new PineconeVectorStore(this.Client);
-
- var hotelRecordDefinition = new VectorStoreRecordDefinition
- {
- Properties =
- [
- new VectorStoreRecordKeyProperty(nameof(PineconeHotel.HotelId), typeof(string)),
- new VectorStoreRecordDataProperty(nameof(PineconeHotel.HotelName), typeof(string)),
- new VectorStoreRecordDataProperty(nameof(PineconeHotel.HotelCode), typeof(int)),
- new VectorStoreRecordDataProperty(nameof(PineconeHotel.ParkingIncluded), typeof(bool)) { StoragePropertyName = "parking_is_included" },
- new VectorStoreRecordDataProperty(nameof(PineconeHotel.HotelRating), typeof(float)),
- new VectorStoreRecordDataProperty(nameof(PineconeHotel.Tags), typeof(List)),
- new VectorStoreRecordDataProperty(nameof(PineconeHotel.Description), typeof(string)),
- new VectorStoreRecordVectorProperty(nameof(PineconeHotel.DescriptionEmbedding), typeof(ReadOnlyMemory)) { Dimensions = 8, DistanceFunction = DistanceFunction.DotProductSimilarity }
- ]
- };
-
- var allTypesRecordDefinition = new VectorStoreRecordDefinition
- {
- Properties =
- [
- new VectorStoreRecordKeyProperty(nameof(PineconeAllTypes.Id), typeof(string)),
- new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.BoolProperty), typeof(bool)),
- new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableBoolProperty), typeof(bool?)),
- new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.StringProperty), typeof(string)),
- new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableStringProperty), typeof(string)),
- new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.IntProperty), typeof(int)),
- new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableIntProperty), typeof(int?)),
- new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.LongProperty), typeof(long)),
- new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableLongProperty), typeof(long?)),
- new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.FloatProperty), typeof(float)),
- new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableFloatProperty), typeof(float?)),
- new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.DoubleProperty), typeof(double)),
- new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableDoubleProperty), typeof(double?)),
- new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.DecimalProperty), typeof(decimal)),
- new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableDecimalProperty), typeof(decimal?)),
- new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.StringArray), typeof(string[])),
- new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableStringArray), typeof(string[])),
- new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.StringList), typeof(List)),
- new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableStringList), typeof(List)),
- new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.Collection), typeof(IReadOnlyCollection)),
- new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.Enumerable), typeof(IEnumerable)),
- new VectorStoreRecordVectorProperty(nameof(PineconeAllTypes.Embedding), typeof(ReadOnlyMemory?)) { Dimensions = 8, DistanceFunction = DistanceFunction.DotProductSimilarity }
- ]
- };
-
- this.HotelRecordCollection = new PineconeVectorStoreRecordCollection(
- this.Client,
- this.IndexName,
- new PineconeVectorStoreRecordCollectionOptions
- {
- VectorStoreRecordDefinition = hotelRecordDefinition
- });
-
- this.AllTypesRecordCollection = new PineconeVectorStoreRecordCollection(
- this.Client,
- this.IndexName,
- new PineconeVectorStoreRecordCollectionOptions
- {
- VectorStoreRecordDefinition = allTypesRecordDefinition
- });
-
- this.HotelRecordCollectionWithCustomNamespace = new PineconeVectorStoreRecordCollection(
- this.Client,
- this.IndexName,
- new PineconeVectorStoreRecordCollectionOptions
- {
- VectorStoreRecordDefinition = hotelRecordDefinition,
- IndexNamespace = "my-namespace"
- });
-
- this.HotelRecordCollectionFromVectorStore = this.VectorStore.GetCollection(
- this.IndexName,
- hotelRecordDefinition);
-
- this.HotelRecordCollectionWithGenericDataModel = this.VectorStore.GetCollection>(
- this.IndexName,
- hotelRecordDefinition);
-
- await this.ClearIndexesAsync();
- await this.CreateIndexAndWaitAsync();
- await this.AddSampleDataAsync();
- }
-
- private async Task CreateIndexAndWaitAsync()
- {
- var attemptCount = 0;
-
- await this.HotelRecordCollection.CreateCollectionAsync();
-
- do
- {
- await Task.Delay(DelayInterval);
- attemptCount++;
- this.Index = await this.Client.GetIndex(this.IndexName);
- } while (!this.Index.Status.IsReady && attemptCount <= MaxAttemptCount);
-
- if (!this.Index.Status.IsReady)
- {
- throw new InvalidOperationException("'Create index' operation didn't complete in time. Index name: " + this.IndexName);
- }
- }
-
- public async Task DisposeAsync()
- {
- if (this.Client is not null)
- {
- await this.ClearIndexesAsync();
- this.Client.Dispose();
- }
- }
-
- private async Task AddSampleDataAsync()
- {
- var fiveSeasons = new PineconeHotel
- {
- HotelId = "five-seasons",
- HotelName = "Five Seasons Hotel",
- Description = "Great service any season.",
- HotelCode = 7,
- HotelRating = 4.5f,
- ParkingIncluded = true,
- DescriptionEmbedding = new ReadOnlyMemory([7.5f, 71.0f, 71.5f, 72.0f, 72.5f, 73.0f, 73.5f, 74.0f]),
- Tags = ["wi-fi", "sauna", "gym", "pool"]
- };
-
- var vacationInn = new PineconeHotel
- {
- HotelId = "vacation-inn",
- HotelName = "Vacation Inn Hotel",
- Description = "On vacation? Stay with us.",
- HotelCode = 11,
- HotelRating = 4.3f,
- ParkingIncluded = true,
- DescriptionEmbedding = new ReadOnlyMemory([17.5f, 721.0f, 731.5f, 742.0f, 762.5f, 783.0f, 793.5f, 704.0f]),
- Tags = ["wi-fi", "breakfast", "gym"]
- };
-
- var bestEastern = new PineconeHotel
- {
- HotelId = "best-eastern",
- HotelName = "Best Eastern Hotel",
- Description = "Best hotel east of New York.",
- HotelCode = 42,
- HotelRating = 4.7f,
- ParkingIncluded = true,
- DescriptionEmbedding = new ReadOnlyMemory([47.5f, 421.0f, 741.5f, 744.0f, 742.5f, 483.0f, 743.5f, 744.0f]),
- Tags = ["wi-fi", "breakfast", "gym"]
- };
-
- var stats = await this.Index.DescribeStats();
- var vectorCountBefore = stats.TotalVectorCount;
-
- // use both Upsert and BatchUpsert methods and also use record collections created directly and using vector store
- await this.HotelRecordCollection.UpsertAsync(fiveSeasons);
- vectorCountBefore = await this.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: 1);
-
- await this.HotelRecordCollectionFromVectorStore.UpsertBatchAsync([vacationInn, bestEastern]).ToListAsync();
- vectorCountBefore = await this.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: 2);
-
- var allTypes1 = new PineconeAllTypes
- {
- Id = "all-types-1",
- BoolProperty = true,
- NullableBoolProperty = false,
- StringProperty = "string prop 1",
- NullableStringProperty = "nullable prop 1",
- IntProperty = 1,
- NullableIntProperty = 10,
- LongProperty = 100L,
- NullableLongProperty = 1000L,
- FloatProperty = 10.5f,
- NullableFloatProperty = 100.5f,
- DoubleProperty = 23.75d,
- NullableDoubleProperty = 233.75d,
- DecimalProperty = 50.75m,
- NullableDecimalProperty = 500.75m,
- StringArray = ["one", "two"],
- NullableStringArray = ["five", "six"],
- StringList = ["eleven", "twelve"],
- NullableStringList = ["fifteen", "sixteen"],
- Collection = ["Foo", "Bar"],
- Enumerable = ["another", "and another"],
- Embedding = new ReadOnlyMemory([1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f, 8.5f])
- };
-
- var allTypes2 = new PineconeAllTypes
- {
- Id = "all-types-2",
- BoolProperty = false,
- NullableBoolProperty = null,
- StringProperty = "string prop 2",
- NullableStringProperty = null,
- IntProperty = 2,
- NullableIntProperty = null,
- LongProperty = 200L,
- NullableLongProperty = null,
- FloatProperty = 20.5f,
- NullableFloatProperty = null,
- DoubleProperty = 43.75,
- NullableDoubleProperty = null,
- DecimalProperty = 250.75M,
- NullableDecimalProperty = null,
- StringArray = [],
- NullableStringArray = null,
- StringList = [],
- NullableStringList = null,
- Collection = [],
- Enumerable = [],
- Embedding = new ReadOnlyMemory([10.5f, 20.5f, 30.5f, 40.5f, 50.5f, 60.5f, 70.5f, 80.5f])
- };
-
- await this.AllTypesRecordCollection.UpsertBatchAsync([allTypes1, allTypes2]).ToListAsync();
- vectorCountBefore = await this.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: 2);
-
- var custom = new PineconeHotel
- {
- HotelId = "custom-hotel",
- HotelName = "Custom Hotel",
- Description = "Everything customizable!",
- HotelCode = 17,
- HotelRating = 4.25f,
- ParkingIncluded = true,
- DescriptionEmbedding = new ReadOnlyMemory([147.5f, 1421.0f, 1741.5f, 1744.0f, 1742.5f, 1483.0f, 1743.5f, 1744.0f]),
- };
-
- await this.HotelRecordCollectionWithCustomNamespace.UpsertAsync(custom);
- vectorCountBefore = await this.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: 1);
- }
-
- public async Task VerifyVectorCountModifiedAsync(uint vectorCountBefore, int delta)
- {
- var attemptCount = 0;
- Sdk.IndexStats stats;
-
- do
- {
- await Task.Delay(DelayInterval);
- attemptCount++;
- stats = await this.Index.DescribeStats();
- } while (stats.TotalVectorCount != vectorCountBefore + delta && attemptCount <= MaxAttemptCount);
-
- if (stats.TotalVectorCount != vectorCountBefore + delta)
- {
- throw new InvalidOperationException("'Upsert'/'Delete' operation didn't complete in time.");
- }
-
- return stats.TotalVectorCount;
- }
-
- public async Task DeleteAndWaitAsync(IEnumerable ids, string? indexNamespace = null)
- {
- var stats = await this.Index.DescribeStats();
- var vectorCountBefore = stats.Namespaces.Single(x => x.Name == (indexNamespace ?? "")).VectorCount;
- var idCount = ids.Count();
-
- var attemptCount = 0;
- await this.Index.Delete(ids, indexNamespace);
- long vectorCount;
- do
- {
- await Task.Delay(DelayInterval);
- attemptCount++;
- stats = await this.Index.DescribeStats();
- vectorCount = stats.Namespaces.Single(x => x.Name == (indexNamespace ?? "")).VectorCount;
- } while (vectorCount > vectorCountBefore - idCount && attemptCount <= MaxAttemptCount);
-
- if (vectorCount > vectorCountBefore - idCount)
- {
- throw new InvalidOperationException("'Delete' operation didn't complete in time.");
- }
- }
-
- private async Task ClearIndexesAsync()
- {
- var indexes = await this.Client.ListIndexes();
- var deletions = indexes.Select(x => this.DeleteExistingIndexAndWaitAsync(x.Name));
-
- await Task.WhenAll(deletions);
- }
-
- private async Task DeleteExistingIndexAndWaitAsync(string indexName)
- {
- var exists = true;
- try
- {
- var attemptCount = 0;
- await this.Client.DeleteIndex(indexName);
-
- do
- {
- await Task.Delay(DelayInterval);
- var indexes = (await this.Client.ListIndexes()).Select(x => x.Name).ToArray();
- if (indexes.Length == 0 || !indexes.Contains(indexName))
- {
- exists = false;
- }
- } while (exists && attemptCount <= MaxAttemptCount);
- }
- catch (HttpRequestException ex) when (ex.Message.Contains("NOT_FOUND"))
- {
- // index was already deleted
- exists = false;
- }
-
- if (exists)
- {
- throw new InvalidOperationException("'Delete index' operation didn't complete in time. Index name: " + indexName);
- }
- }
-}
diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreRecordCollectionTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreRecordCollectionTests.cs
deleted file mode 100644
index 9b68eaf8d863..000000000000
--- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreRecordCollectionTests.cs
+++ /dev/null
@@ -1,684 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System;
-using System.Linq;
-using System.Net;
-using System.Net.Http;
-using System.Threading.Tasks;
-using Grpc.Core;
-using Microsoft.Extensions.VectorData;
-using Microsoft.SemanticKernel.Connectors.Pinecone;
-using Pinecone;
-using SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone.Xunit;
-using SemanticKernel.IntegrationTests.Connectors.Memory.Xunit;
-using Xunit;
-
-namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone;
-
-#pragma warning disable CS0618 // VectorSearchFilter is obsolete
-
-[Collection("PineconeVectorStoreTests")]
-[PineconeApiKeySetCondition]
-public class PineconeVectorStoreRecordCollectionTests(PineconeVectorStoreFixture fixture) : IClassFixture
-{
- private PineconeVectorStoreFixture Fixture { get; } = fixture;
-
- [VectorStoreFact]
- public async Task TryCreateExistingIndexIsNoopAsync()
- {
- await this.Fixture.HotelRecordCollection.CreateCollectionIfNotExistsAsync();
- }
-
- [VectorStoreFact]
- public async Task CollectionExistsReturnsTrueForExistingCollectionAsync()
- {
- var result = await this.Fixture.HotelRecordCollection.CollectionExistsAsync();
-
- Assert.True(result);
- }
-
- [VectorStoreTheory]
- [InlineData(true)]
- [InlineData(false)]
- public async Task BasicGetAsync(bool includeVectors)
- {
- var fiveSeasons = await this.Fixture.HotelRecordCollection.GetAsync("five-seasons", new GetRecordOptions { IncludeVectors = includeVectors });
-
- Assert.NotNull(fiveSeasons);
- Assert.Equal("five-seasons", fiveSeasons.HotelId);
- Assert.Equal("Five Seasons Hotel", fiveSeasons.HotelName);
- Assert.Equal("Great service any season.", fiveSeasons.Description);
- Assert.Equal(7, fiveSeasons.HotelCode);
- Assert.Equal(4.5f, fiveSeasons.HotelRating);
- Assert.True(fiveSeasons.ParkingIncluded);
- Assert.Contains("wi-fi", fiveSeasons.Tags);
- Assert.Contains("sauna", fiveSeasons.Tags);
- Assert.Contains("gym", fiveSeasons.Tags);
- Assert.Contains("pool", fiveSeasons.Tags);
-
- if (includeVectors)
- {
- Assert.Equal(new ReadOnlyMemory([7.5f, 71.0f, 71.5f, 72.0f, 72.5f, 73.0f, 73.5f, 74.0f]), fiveSeasons.DescriptionEmbedding);
- }
- else
- {
- Assert.Equal(new ReadOnlyMemory([]), fiveSeasons.DescriptionEmbedding);
- }
- }
-
- [VectorStoreTheory]
- [InlineData(true)]
- [InlineData(false)]
- public async Task BatchGetAsync(bool collectionFromVectorStore)
- {
- var hotelsCollection = collectionFromVectorStore
- ? this.Fixture.HotelRecordCollection
- : this.Fixture.HotelRecordCollectionFromVectorStore;
-
- var hotels = await hotelsCollection.GetBatchAsync(["five-seasons", "vacation-inn", "best-eastern"]).ToListAsync();
-
- var fiveSeasons = hotels.Single(x => x.HotelId == "five-seasons");
- var vacationInn = hotels.Single(x => x.HotelId == "vacation-inn");
- var bestEastern = hotels.Single(x => x.HotelId == "best-eastern");
-
- Assert.Equal("Five Seasons Hotel", fiveSeasons.HotelName);
- Assert.Equal("Great service any season.", fiveSeasons.Description);
- Assert.Equal(7, fiveSeasons.HotelCode);
- Assert.Equal(4.5f, fiveSeasons.HotelRating);
- Assert.True(fiveSeasons.ParkingIncluded);
- Assert.Contains("wi-fi", fiveSeasons.Tags);
- Assert.Contains("sauna", fiveSeasons.Tags);
- Assert.Contains("gym", fiveSeasons.Tags);
- Assert.Contains("pool", fiveSeasons.Tags);
-
- Assert.Equal("Vacation Inn Hotel", vacationInn.HotelName);
- Assert.Equal("On vacation? Stay with us.", vacationInn.Description);
- Assert.Equal(11, vacationInn.HotelCode);
- Assert.Equal(4.3f, vacationInn.HotelRating);
- Assert.True(vacationInn.ParkingIncluded);
- Assert.Contains("wi-fi", vacationInn.Tags);
- Assert.Contains("breakfast", vacationInn.Tags);
- Assert.Contains("gym", vacationInn.Tags);
-
- Assert.Equal("Best Eastern Hotel", bestEastern.HotelName);
- Assert.Equal("Best hotel east of New York.", bestEastern.Description);
- Assert.Equal(42, bestEastern.HotelCode);
- Assert.Equal(4.7f, bestEastern.HotelRating);
- Assert.True(bestEastern.ParkingIncluded);
- Assert.Contains("wi-fi", bestEastern.Tags);
- Assert.Contains("breakfast", bestEastern.Tags);
- Assert.Contains("gym", bestEastern.Tags);
- }
-
- [VectorStoreTheory]
- [InlineData(true)]
- [InlineData(false)]
- public async Task AllTypesBatchGetAsync(bool includeVectors)
- {
- var allTypes = await this.Fixture.AllTypesRecordCollection.GetBatchAsync(["all-types-1", "all-types-2"], new GetRecordOptions { IncludeVectors = includeVectors }).ToListAsync();
-
- var allTypes1 = allTypes.Single(x => x.Id == "all-types-1");
- var allTypes2 = allTypes.Single(x => x.Id == "all-types-2");
-
- Assert.True(allTypes1.BoolProperty);
- Assert.Equal("string prop 1", allTypes1.StringProperty);
- Assert.Equal(1, allTypes1.IntProperty);
- Assert.Equal(100L, allTypes1.LongProperty);
- Assert.Equal(10.5f, allTypes1.FloatProperty);
- Assert.Equal(23.75d, allTypes1.DoubleProperty);
- Assert.Equal(50.75m, allTypes1.DecimalProperty);
- Assert.Contains("one", allTypes1.StringArray);
- Assert.Contains("two", allTypes1.StringArray);
- Assert.Contains("eleven", allTypes1.StringList);
- Assert.Contains("twelve", allTypes1.StringList);
- Assert.Contains("Foo", allTypes1.Collection);
- Assert.Contains("Bar", allTypes1.Collection);
- Assert.Contains("another", allTypes1.Enumerable);
- Assert.Contains("and another", allTypes1.Enumerable);
-
- Assert.False(allTypes2.BoolProperty);
- Assert.Equal("string prop 2", allTypes2.StringProperty);
- Assert.Equal(2, allTypes2.IntProperty);
- Assert.Equal(200L, allTypes2.LongProperty);
- Assert.Equal(20.5f, allTypes2.FloatProperty);
- Assert.Equal(43.75d, allTypes2.DoubleProperty);
- Assert.Equal(250.75m, allTypes2.DecimalProperty);
- Assert.Empty(allTypes2.StringArray);
- Assert.Empty(allTypes2.StringList);
- Assert.Empty(allTypes2.Collection);
- Assert.Empty(allTypes2.Enumerable);
-
- if (includeVectors)
- {
- Assert.True(allTypes1.Embedding.HasValue);
- Assert.Equal(new ReadOnlyMemory([1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f, 8.5f]), allTypes1.Embedding.Value);
-
- Assert.True(allTypes2.Embedding.HasValue);
- Assert.Equal(new ReadOnlyMemory([10.5f, 20.5f, 30.5f, 40.5f, 50.5f, 60.5f, 70.5f, 80.5f]), allTypes2.Embedding.Value);
- }
- else
- {
- Assert.Null(allTypes1.Embedding);
- Assert.Null(allTypes2.Embedding);
- }
- }
-
- [VectorStoreFact]
- public async Task BatchGetIncludingNonExistingRecordAsync()
- {
- var hotels = await this.Fixture.HotelRecordCollection.GetBatchAsync(["vacation-inn", "non-existing"]).ToListAsync();
-
- Assert.Single(hotels);
- var vacationInn = hotels.Single(x => x.HotelId == "vacation-inn");
-
- Assert.Equal("Vacation Inn Hotel", vacationInn.HotelName);
- Assert.Equal("On vacation? Stay with us.", vacationInn.Description);
- Assert.Equal(11, vacationInn.HotelCode);
- Assert.Equal(4.3f, vacationInn.HotelRating);
- Assert.True(vacationInn.ParkingIncluded);
- Assert.Contains("wi-fi", vacationInn.Tags);
- Assert.Contains("breakfast", vacationInn.Tags);
- Assert.Contains("gym", vacationInn.Tags);
- }
-
- [VectorStoreFact]
- public async Task GetNonExistingRecordAsync()
- {
- var result = await this.Fixture.HotelRecordCollection.GetAsync("non-existing");
- Assert.Null(result);
- }
-
- [VectorStoreTheory]
- [InlineData(true)]
- [InlineData(false)]
- public async Task GetFromCustomNamespaceAsync(bool includeVectors)
- {
- var custom = await this.Fixture.HotelRecordCollectionWithCustomNamespace.GetAsync("custom-hotel", new GetRecordOptions { IncludeVectors = includeVectors });
-
- Assert.NotNull(custom);
- Assert.Equal("custom-hotel", custom.HotelId);
- Assert.Equal("Custom Hotel", custom.HotelName);
- if (includeVectors)
- {
- Assert.Equal(new ReadOnlyMemory([147.5f, 1421.0f, 1741.5f, 1744.0f, 1742.5f, 1483.0f, 1743.5f, 1744.0f]), custom.DescriptionEmbedding);
- }
- else
- {
- Assert.Equal(new ReadOnlyMemory([]), custom.DescriptionEmbedding);
- }
- }
-
- [VectorStoreFact]
- public async Task TryGetVectorLocatedInDefaultNamespaceButLookInCustomNamespaceAsync()
- {
- var badFiveSeasons = await this.Fixture.HotelRecordCollectionWithCustomNamespace.GetAsync("five-seasons");
-
- Assert.Null(badFiveSeasons);
- }
-
- [VectorStoreFact]
- public async Task TryGetVectorLocatedInCustomNamespaceButLookInDefaultNamespaceAsync()
- {
- var badCustomHotel = await this.Fixture.HotelRecordCollection.GetAsync("custom-hotel");
-
- Assert.Null(badCustomHotel);
- }
-
- [VectorStoreFact]
- public async Task DeleteNonExistingRecordAsync()
- {
- await this.Fixture.HotelRecordCollection.DeleteAsync("non-existing");
- }
-
- [VectorStoreFact]
- public async Task TryDeleteExistingVectorLocatedInDefaultNamespaceButUseCustomNamespaceDoesNotDoAnythingAsync()
- {
- await this.Fixture.HotelRecordCollectionWithCustomNamespace.DeleteAsync("five-seasons");
-
- var stillThere = await this.Fixture.HotelRecordCollection.GetAsync("five-seasons");
- Assert.NotNull(stillThere);
- Assert.Equal("five-seasons", stillThere.HotelId);
- }
-
- [VectorStoreFact]
- public async Task TryDeleteExistingVectorLocatedInCustomNamespaceButUseDefaultNamespaceDoesNotDoAnythingAsync()
- {
- await this.Fixture.HotelRecordCollection.DeleteAsync("custom-hotel");
-
- var stillThere = await this.Fixture.HotelRecordCollectionWithCustomNamespace.GetAsync("custom-hotel");
- Assert.NotNull(stillThere);
- Assert.Equal("custom-hotel", stillThere.HotelId);
- }
-
- [VectorStoreTheory]
- [InlineData(true)]
- [InlineData(false)]
- public async Task InsertGetModifyDeleteVectorAsync(bool collectionFromVectorStore)
- {
- var langriSha = new PineconeHotel
- {
- HotelId = "langri-sha",
- HotelName = "Langri-Sha Hotel",
- Description = "Lorem ipsum",
- HotelCode = 100,
- HotelRating = 4.2f,
- ParkingIncluded = false,
- DescriptionEmbedding = new ReadOnlyMemory([1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f])
- };
-
- var stats = await this.Fixture.Index.DescribeStats();
- var vectorCountBefore = stats.TotalVectorCount;
-
- var hotelRecordCollection = collectionFromVectorStore
- ? this.Fixture.HotelRecordCollectionFromVectorStore
- : this.Fixture.HotelRecordCollection;
-
- // insert
- await hotelRecordCollection.UpsertAsync(langriSha);
-
- vectorCountBefore = await this.Fixture.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: 1);
-
- var inserted = await hotelRecordCollection.GetAsync("langri-sha", new GetRecordOptions { IncludeVectors = true });
-
- Assert.NotNull(inserted);
- Assert.Equal(langriSha.HotelName, inserted.HotelName);
- Assert.Equal(langriSha.Description, inserted.Description);
- Assert.Equal(langriSha.HotelCode, inserted.HotelCode);
- Assert.Equal(langriSha.HotelRating, inserted.HotelRating);
- Assert.Equal(langriSha.ParkingIncluded, inserted.ParkingIncluded);
- Assert.Equal(langriSha.DescriptionEmbedding, inserted.DescriptionEmbedding);
-
- langriSha.Description += " dolor sit amet";
- langriSha.ParkingIncluded = true;
- langriSha.DescriptionEmbedding = new ReadOnlyMemory([11f, 12f, 13f, 14f, 15f, 16f, 17f, 18f]);
-
- // update
- await hotelRecordCollection.UpsertAsync(langriSha);
-
- // this is not great but no vectors are added so we can't query status for number of vectors like we do for insert/delete
- await Task.Delay(2000);
-
- var updated = await hotelRecordCollection.GetAsync("langri-sha", new GetRecordOptions { IncludeVectors = true });
-
- Assert.NotNull(updated);
- Assert.Equal(langriSha.HotelName, updated.HotelName);
- Assert.Equal(langriSha.Description, updated.Description);
- Assert.Equal(langriSha.HotelCode, updated.HotelCode);
- Assert.Equal(langriSha.HotelRating, updated.HotelRating);
- Assert.Equal(langriSha.ParkingIncluded, updated.ParkingIncluded);
- Assert.Equal(langriSha.DescriptionEmbedding, updated.DescriptionEmbedding);
-
- // delete
- await hotelRecordCollection.DeleteAsync("langri-sha");
-
- await this.Fixture.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: -1);
- }
-
- [VectorStoreTheory]
- [InlineData(true, true)]
- [InlineData(true, false)]
- [InlineData(false, true)]
- [InlineData(false, false)]
- public async Task VectorizedSearchAsync(bool collectionFromVectorStore, bool includeVectors)
- {
- // Arrange.
- var hotelRecordCollection = collectionFromVectorStore
- ? this.Fixture.HotelRecordCollectionFromVectorStore
- : this.Fixture.HotelRecordCollection;
- var searchVector = new ReadOnlyMemory([17.5f, 721.0f, 731.5f, 742.0f, 762.5f, 783.0f, 793.5f, 704.0f]);
-
- // Act.
- var actual = await hotelRecordCollection.VectorizedSearchAsync(searchVector, new() { IncludeVectors = includeVectors });
- var searchResults = await actual.Results.ToListAsync();
- var searchResultRecord = searchResults.First().Record;
-
- Assert.Equal("Vacation Inn Hotel", searchResultRecord.HotelName);
- Assert.Equal("On vacation? Stay with us.", searchResultRecord.Description);
- Assert.Equal(11, searchResultRecord.HotelCode);
- Assert.Equal(4.3f, searchResultRecord.HotelRating);
- Assert.True(searchResultRecord.ParkingIncluded);
- Assert.Contains("wi-fi", searchResultRecord.Tags);
- Assert.Contains("breakfast", searchResultRecord.Tags);
- Assert.Contains("gym", searchResultRecord.Tags);
- Assert.Equal(includeVectors, searchResultRecord.DescriptionEmbedding.Length > 0);
- }
-
- [VectorStoreTheory]
- [InlineData(true)]
- [InlineData(false)]
- public async Task VectorizedSearchWithTopSkipAsync(bool collectionFromVectorStore)
- {
- // Arrange.
- var hotelRecordCollection = collectionFromVectorStore
- ? this.Fixture.HotelRecordCollectionFromVectorStore
- : this.Fixture.HotelRecordCollection;
- var searchVector = new ReadOnlyMemory([17.5f, 721.0f, 731.5f, 742.0f, 762.5f, 783.0f, 793.5f, 704.0f]);
-
- // Act.
- var actual = await hotelRecordCollection.VectorizedSearchAsync(searchVector, new() { Skip = 1, Top = 1 });
- var searchResults = await actual.Results.ToListAsync();
- Assert.Single(searchResults);
- var searchResultRecord = searchResults.First().Record;
- Assert.Equal("Best Eastern Hotel", searchResultRecord.HotelName);
- }
-
- [VectorStoreTheory]
- [InlineData(true)]
- [InlineData(false)]
- public async Task VectorizedSearchWithFilterAsync(bool collectionFromVectorStore)
- {
- // Arrange.
- var hotelRecordCollection = collectionFromVectorStore
- ? this.Fixture.HotelRecordCollectionFromVectorStore
- : this.Fixture.HotelRecordCollection;
- var searchVector = new ReadOnlyMemory([17.5f, 721.0f, 731.5f, 742.0f, 762.5f, 783.0f, 793.5f, 704.0f]);
-
- // Act.
- var filter = new VectorSearchFilter().EqualTo(nameof(PineconeHotel.HotelCode), 42);
- var actual = await hotelRecordCollection.VectorizedSearchAsync(searchVector, new() { Top = 1, OldFilter = filter });
- var searchResults = await actual.Results.ToListAsync();
- Assert.Single(searchResults);
- var searchResultRecord = searchResults.First().Record;
- Assert.Equal("Best Eastern Hotel", searchResultRecord.HotelName);
- }
-
- [VectorStoreFact]
- public async Task ItCanUpsertAndRetrieveUsingTheGenericMapperAsync()
- {
- var merryYacht = new VectorStoreGenericDataModel("merry-yacht")
- {
- Data =
- {
- ["HotelName"] = "Merry Yacht Hotel",
- ["Description"] = "Stay afloat at the Merry Yacht Hotel",
- ["HotelCode"] = 101,
- ["HotelRating"] = 4.2f,
- ["ParkingIncluded"] = true,
- ["Tags"] = new[] { "wi-fi", "breakfast", "gym" }
- },
- Vectors =
- {
- ["DescriptionEmbedding"] = new ReadOnlyMemory([1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f])
- }
- };
-
- var stats = await this.Fixture.Index.DescribeStats();
- var vectorCountBefore = stats.TotalVectorCount;
-
- var hotelRecordCollection = this.Fixture.HotelRecordCollectionWithGenericDataModel;
-
- // insert
- await hotelRecordCollection.UpsertAsync(merryYacht);
-
- vectorCountBefore = await this.Fixture.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: 1);
-
- var inserted = await hotelRecordCollection.GetAsync("merry-yacht", new GetRecordOptions { IncludeVectors = true });
-
- Assert.NotNull(inserted);
- Assert.Equal(merryYacht.Data["HotelName"], inserted.Data["HotelName"]);
- Assert.Equal(merryYacht.Data["Description"], inserted.Data["Description"]);
- Assert.Equal(merryYacht.Data["HotelCode"], inserted.Data["HotelCode"]);
- Assert.Equal(merryYacht.Data["HotelRating"], inserted.Data["HotelRating"]);
- Assert.Equal(merryYacht.Data["ParkingIncluded"], inserted.Data["ParkingIncluded"]);
- Assert.Equal(merryYacht.Data["Tags"], inserted.Data["Tags"]);
- Assert.Equal(
- ((ReadOnlyMemory)merryYacht.Vectors["DescriptionEmbedding"]!).ToArray(),
- ((ReadOnlyMemory)inserted.Vectors["DescriptionEmbedding"]!).ToArray());
-
- // delete
- await hotelRecordCollection.DeleteAsync("merry-yacht");
-
- await this.Fixture.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: -1);
- }
-
- [VectorStoreFact]
- public async Task UseCollectionExistsOnNonExistingStoreReturnsFalseAsync()
- {
- var incorrectRecordStore = new PineconeVectorStoreRecordCollection(
- this.Fixture.Client,
- "incorrect");
-
- var result = await incorrectRecordStore.CollectionExistsAsync();
-
- Assert.False(result);
- }
-
- [VectorStoreFact]
- public async Task UseNonExistingIndexThrowsAsync()
- {
- var incorrectRecordStore = new PineconeVectorStoreRecordCollection(
- this.Fixture.Client,
- "incorrect");
-
- var statusCode = (await Assert.ThrowsAsync(
- () => incorrectRecordStore.GetAsync("best-eastern"))).StatusCode;
-
- Assert.Equal(HttpStatusCode.NotFound, statusCode);
- }
-
- [VectorStoreFact]
- public async Task UseRecordStoreWithCustomMapperAsync()
- {
- var recordStore = new PineconeVectorStoreRecordCollection(
- this.Fixture.Client,
- this.Fixture.IndexName,
- new PineconeVectorStoreRecordCollectionOptions { VectorCustomMapper = new CustomHotelRecordMapper() });
-
- var vacationInn = await recordStore.GetAsync("vacation-inn", new GetRecordOptions { IncludeVectors = true });
-
- Assert.NotNull(vacationInn);
- Assert.Equal("Custom Vacation Inn Hotel", vacationInn.HotelName);
- Assert.Equal("On vacation? Stay with us.", vacationInn.Description);
- Assert.Equal(11, vacationInn.HotelCode);
- Assert.Equal(4.3f, vacationInn.HotelRating);
- Assert.True(vacationInn.ParkingIncluded);
- Assert.Contains("wi-fi", vacationInn.Tags);
- Assert.Contains("breakfast", vacationInn.Tags);
- Assert.Contains("gym", vacationInn.Tags);
- }
-
- private sealed class CustomHotelRecordMapper : IVectorStoreRecordMapper
- {
- public Vector MapFromDataToStorageModel(PineconeHotel dataModel)
- {
- var metadata = new MetadataMap
- {
- [nameof(PineconeHotel.HotelName)] = dataModel.HotelName,
- [nameof(PineconeHotel.Description)] = dataModel.Description,
- [nameof(PineconeHotel.HotelCode)] = dataModel.HotelCode,
- [nameof(PineconeHotel.HotelRating)] = dataModel.HotelRating,
- ["parking_is_included"] = dataModel.ParkingIncluded,
- [nameof(PineconeHotel.Tags)] = dataModel.Tags.ToArray(),
- };
-
- return new Vector
- {
- Id = dataModel.HotelId,
- Values = dataModel.DescriptionEmbedding.ToArray(),
- Metadata = metadata,
- };
- }
-
- public PineconeHotel MapFromStorageToDataModel(Vector storageModel, StorageToDataModelMapperOptions options)
- {
- if (storageModel.Metadata == null)
- {
- throw new InvalidOperationException("Missing metadata.");
- }
-
- return new PineconeHotel
- {
- HotelId = storageModel.Id,
- HotelName = "Custom " + (string)storageModel.Metadata[nameof(PineconeHotel.HotelName)].Inner!,
- Description = (string)storageModel.Metadata[nameof(PineconeHotel.Description)].Inner!,
- HotelCode = (int)(double)storageModel.Metadata[nameof(PineconeHotel.HotelCode)].Inner!,
- HotelRating = (float)(double)storageModel.Metadata[nameof(PineconeHotel.HotelRating)].Inner!,
- ParkingIncluded = (bool)storageModel.Metadata["parking_is_included"].Inner!,
- Tags = ((MetadataValue[])storageModel.Metadata[nameof(PineconeHotel.Tags)].Inner!)!.Select(x => (string)x.Inner!).ToList(),
- };
- }
- }
-
- #region Negative
-
- [VectorStoreFact]
- public void UseRecordWithNoEmbeddingThrows()
- {
- var exception = Assert.Throws(
- () => new PineconeVectorStoreRecordCollection(
- this.Fixture.Client,
- "Whatever"));
-
- Assert.Equal(
- $"No vector property found on type {nameof(PineconeRecordNoEmbedding)} or the provided VectorStoreRecordDefinition while at least one is required.",
- exception.Message);
- }
-
-#pragma warning disable CA1812
- private sealed record PineconeRecordNoEmbedding
- {
- [VectorStoreRecordKey]
- public int Id { get; set; }
-
- [VectorStoreRecordData]
- public string? Name { get; set; }
- }
-#pragma warning restore CA1812
-
- [VectorStoreFact]
- public void UseRecordWithMultipleEmbeddingsThrows()
- {
- var exception = Assert.Throws(
- () => new PineconeVectorStoreRecordCollection(
- this.Fixture.Client,
- "Whatever"));
-
- Assert.Equal(
- $"Multiple vector properties found on type {nameof(PineconeRecordMultipleEmbeddings)} or the provided VectorStoreRecordDefinition while only one is supported.",
- exception.Message);
- }
-
-#pragma warning disable CA1812
- private sealed record PineconeRecordMultipleEmbeddings
- {
- [VectorStoreRecordKey]
- public string Id { get; set; } = null!;
-
- [VectorStoreRecordVector]
- public ReadOnlyMemory Embedding1 { get; set; }
-
- [VectorStoreRecordVector]
- public ReadOnlyMemory Embedding2 { get; set; }
- }
-#pragma warning restore CA1812
-
- [VectorStoreFact]
- public void UseRecordWithUnsupportedKeyTypeThrows()
- {
- var message = Assert.Throws(
- () => new PineconeVectorStoreRecordCollection(
- this.Fixture.Client,
- "Whatever")).Message;
-
- Assert.Equal(
- $"Key properties must be one of the supported types: {typeof(string).FullName}. Type of the property '{nameof(PineconeRecordUnsupportedKeyType.Id)}' is {typeof(int).FullName}.",
- message);
- }
-
-#pragma warning disable CA1812
- private sealed record PineconeRecordUnsupportedKeyType
- {
- [VectorStoreRecordKey]
- public int Id { get; set; }
-
- [VectorStoreRecordData]
- public string? Name { get; set; }
-
- [VectorStoreRecordVector]
- public ReadOnlyMemory Embedding { get; set; }
- }
-#pragma warning restore CA1812
-
- [VectorStoreFact]
- public async Task TryAddingVectorWithUnsupportedValuesAsync()
- {
- var badAllTypes = new PineconeAllTypes
- {
- Id = "bad",
- BoolProperty = true,
- DecimalProperty = 1m,
- DoubleProperty = 1.5d,
- FloatProperty = 2.5f,
- IntProperty = 1,
- LongProperty = 11L,
- NullableStringArray = ["foo", null!, "bar",],
- Embedding = new ReadOnlyMemory([1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f])
- };
-
- var exception = await Assert.ThrowsAsync(
- () => this.Fixture.AllTypesRecordCollection.UpsertAsync(badAllTypes));
-
- Assert.Equal("Microsoft.SemanticKernel.Connectors.Pinecone", exception.Source);
- Assert.Equal("Pinecone", exception.VectorStoreType);
- Assert.Equal("Upsert", exception.OperationName);
- Assert.Equal(this.Fixture.IndexName, exception.CollectionName);
-
- var inner = exception.InnerException as RpcException;
- Assert.NotNull(inner);
- Assert.Equal(StatusCode.InvalidArgument, inner.StatusCode);
- }
-
- [VectorStoreFact]
- public async Task TryCreateIndexWithIncorrectDimensionFailsAsync()
- {
- var recordCollection = new PineconeVectorStoreRecordCollection(
- this.Fixture.Client,
- "negative-dimension");
-
- var message = (await Assert.ThrowsAsync(() => recordCollection.CreateCollectionAsync())).Message;
-
- Assert.Equal("Property Dimensions on VectorStoreRecordVectorProperty 'Embedding' must be set to a positive integer to create a collection.", message);
- }
-
-#pragma warning disable CA1812
- private sealed record PineconeRecordWithIncorrectDimension
- {
- [VectorStoreRecordKey]
- public string Id { get; set; } = null!;
-
- [VectorStoreRecordData]
- public string? Name { get; set; }
-
- [VectorStoreRecordVector(Dimensions: -7)]
- public ReadOnlyMemory Embedding { get; set; }
- }
-#pragma warning restore CA1812
-
- [VectorStoreFact]
- public async Task TryCreateIndexWithUnsSupportedMetricFailsAsync()
- {
- var recordCollection = new PineconeVectorStoreRecordCollection(
- this.Fixture.Client,
- "bad-metric");
-
- var message = (await Assert.ThrowsAsync(() => recordCollection.CreateCollectionAsync())).Message;
-
- Assert.Equal("Distance function 'just eyeball it' for VectorStoreRecordVectorProperty 'Embedding' is not supported by the Pinecone VectorStore.", message);
- }
-
-#pragma warning disable CA1812
- private sealed record PineconeRecordWithUnsupportedMetric
- {
- [VectorStoreRecordKey]
- public string Id { get; set; } = null!;
-
- [VectorStoreRecordData]
- public string? Name { get; set; }
-
- [VectorStoreRecordVector(Dimensions: 5, DistanceFunction: "just eyeball it")]
- public ReadOnlyMemory Embedding { get; set; }
- }
-#pragma warning restore CA1812
-
- #endregion
-}
diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreTests.cs
deleted file mode 100644
index 4cd63ec6b8a9..000000000000
--- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreTests.cs
+++ /dev/null
@@ -1,54 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System;
-using Microsoft.Extensions.VectorData;
-using Microsoft.SemanticKernel.Connectors.Pinecone;
-using SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone.Xunit;
-using SemanticKernel.IntegrationTests.Connectors.Memory.Xunit;
-using Xunit;
-using Sdk = Pinecone;
-
-namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone;
-
-[Collection("PineconeVectorStoreTests")]
-[PineconeApiKeySetCondition]
-public class PineconeVectorStoreTests(PineconeVectorStoreFixture fixture)
- : BaseVectorStoreTests(new PineconeVectorStore(fixture.Client)), IClassFixture
-{
- private PineconeVectorStoreFixture Fixture { get; } = fixture;
-
-#pragma warning disable CS0618 // IPineconeVectorStoreRecordCollectionFactory is obsolete
- [VectorStoreFact]
- public void CreateCollectionUsingFactory()
- {
- var vectorStore = new PineconeVectorStore(
- this.Fixture.Client,
- new PineconeVectorStoreOptions
- {
- VectorStoreCollectionFactory = new MyVectorStoreRecordCollectionFactory()
- });
-
- var factoryCollection = vectorStore.GetCollection(this.Fixture.IndexName);
-
- Assert.NotNull(factoryCollection);
- Assert.Equal("factory" + this.Fixture.IndexName, factoryCollection.CollectionName);
- }
-
- private sealed class MyVectorStoreRecordCollectionFactory : IPineconeVectorStoreRecordCollectionFactory
- {
- public IVectorStoreRecordCollection CreateVectorStoreRecordCollection(
- Sdk.PineconeClient pineconeClient,
- string name,
- VectorStoreRecordDefinition? vectorStoreRecordDefinition)
- where TKey : notnull
- {
- if (typeof(TKey) != typeof(string))
- {
- throw new InvalidOperationException("Only string keys are supported.");
- }
-
- return (new PineconeVectorStoreRecordCollection(pineconeClient, "factory" + name) as IVectorStoreRecordCollection)!;
- }
- }
-#pragma warning restore CS0618
-}
diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/Xunit/PineconeApiKeySetConditionAttribute.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/Xunit/PineconeApiKeySetConditionAttribute.cs
deleted file mode 100644
index b677c47c378f..000000000000
--- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/Xunit/PineconeApiKeySetConditionAttribute.cs
+++ /dev/null
@@ -1,21 +0,0 @@
-// Copyright (c) Microsoft. All rights reserved.
-
-using System;
-using System.Threading.Tasks;
-using SemanticKernel.IntegrationTests.Connectors.Memory.Xunit;
-
-namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone.Xunit;
-
-[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class)]
-public sealed class PineconeApiKeySetConditionAttribute : Attribute, ITestCondition
-{
- public ValueTask IsMetAsync()
- {
- var isMet = PineconeUserSecretsExtensions.ContainsPineconeApiKey();
-
- return ValueTask.FromResult(isMet);
- }
-
- public string SkipReason
- => $"Pinecone API key was not specified in user secrets. Use the following command to set it: dotnet user-secrets set \"{PineconeUserSecretsExtensions.PineconeApiKeyUserSecretEntry}\" \"your_Pinecone_API_key\"";
-}
diff --git a/dotnet/src/IntegrationTests/IntegrationTests.csproj b/dotnet/src/IntegrationTests/IntegrationTests.csproj
index 6a75257b503a..033c0f57bd78 100644
--- a/dotnet/src/IntegrationTests/IntegrationTests.csproj
+++ b/dotnet/src/IntegrationTests/IntegrationTests.csproj
@@ -52,7 +52,6 @@
-
@@ -69,7 +68,6 @@
-
diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeBatchConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeBatchConformanceTests.cs
new file mode 100644
index 000000000000..682e43c6d45a
--- /dev/null
+++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeBatchConformanceTests.cs
@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using PineconeIntegrationTests.Support;
+using VectorDataSpecificationTests.CRUD;
+using Xunit;
+
+namespace PineconeIntegrationTests.CRUD;
+
+public class PineconeBatchConformanceTests(PineconeFixture fixture)
+ : BatchConformanceTests(fixture), IClassFixture
+{
+}
diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeCollectionConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeCollectionConformanceTests.cs
new file mode 100644
index 000000000000..45fbbb18dc64
--- /dev/null
+++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeCollectionConformanceTests.cs
@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using PineconeIntegrationTests.Support;
+using VectorDataSpecificationTests.CRUD;
+using Xunit;
+
+namespace PineconeIntegrationTests.CRUD;
+
+public class PineconeCollectionConformanceTests(PineconeFixture fixture)
+ : CollectionConformanceTests(fixture), IClassFixture
+{
+}
diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeGenericDataModelConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeGenericDataModelConformanceTests.cs
new file mode 100644
index 000000000000..0f43f43bec5c
--- /dev/null
+++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeGenericDataModelConformanceTests.cs
@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using PineconeIntegrationTests.Support;
+using VectorDataSpecificationTests.CRUD;
+using Xunit;
+
+namespace PineconeIntegrationTests.CRUD;
+
+public class PineconeGenericDataModelConformanceTests(PineconeFixture fixture)
+ : GenericDataModelConformanceTests(fixture), IClassFixture
+{
+}
diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeRecordConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeRecordConformanceTests.cs
new file mode 100644
index 000000000000..9e4acddea020
--- /dev/null
+++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeRecordConformanceTests.cs
@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using PineconeIntegrationTests.Support;
+using VectorDataSpecificationTests.CRUD;
+using Xunit;
+
+namespace PineconeIntegrationTests.CRUD;
+
+public class PineconeRecordConformanceTests(PineconeFixture fixture)
+ : RecordConformanceTests(fixture), IClassFixture
+{
+}
diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/PineconeIntegrationTests.csproj b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/PineconeIntegrationTests.csproj
new file mode 100644
index 000000000000..bc70e11b12dd
--- /dev/null
+++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/PineconeIntegrationTests.csproj
@@ -0,0 +1,49 @@
+
+
+
+ net8.0;net472
+ enable
+ enable
+
+ false
+ true
+
+ $(NoWarn);CA2007,SKEXP0001,SKEXP0020,VSTHRD111;CS1685
+ b7762d10-e29b-4bb1-8b74-b6d69a667dd4
+
+
+
+
+
+ runtime; build; native; contentfiles; analyzers; buildtransitive
+ all
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Always
+
+
+ Always
+
+
+
+
\ No newline at end of file
diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeFixture.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeFixture.cs
new file mode 100644
index 000000000000..21bbdfd51eaf
--- /dev/null
+++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeFixture.cs
@@ -0,0 +1,13 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Pinecone;
+using VectorDataSpecificationTests.Support;
+
+namespace PineconeIntegrationTests.Support;
+
+public class PineconeFixture : VectorStoreFixture
+{
+ public override TestStore TestStore => PineconeTestStore.Instance;
+
+ public PineconeClient Client => PineconeTestStore.Instance.Client;
+}
diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeTestStore.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeTestStore.cs
new file mode 100644
index 000000000000..509c81c087ed
--- /dev/null
+++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeTestStore.cs
@@ -0,0 +1,88 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using DotNet.Testcontainers.Builders;
+using DotNet.Testcontainers.Containers;
+using Grpc.Net.Client;
+using Microsoft.Extensions.VectorData;
+using Microsoft.SemanticKernel.Connectors.Pinecone;
+using Pinecone;
+using VectorDataSpecificationTests.Support;
+
+namespace PineconeIntegrationTests.Support;
+
+#pragma warning disable CA1001 // Type owns disposable fields but is not disposable
+
+internal sealed class PineconeTestStore : TestStore
+{
+ // Values taken from https://docs.pinecone.io/guides/operations/local-development
+ private const string Image = "ghcr.io/pinecone-io/pinecone-local:v0.7.0";
+ private const ushort RestPort = 5080;
+ private const ushort GrpcPort = RestPort + 1;
+
+ public static PineconeTestStore Instance { get; } = new();
+
+ private IContainer? _container;
+ private Pinecone.PineconeClient? _client;
+ private PineconeVectorStore? _defaultVectorStore;
+
+ public Pinecone.PineconeClient Client => this._client ?? throw new InvalidOperationException("Not initialized");
+
+ public override IVectorStore DefaultVectorStore => this._defaultVectorStore ?? throw new InvalidOperationException("Not initialized");
+
+ private PineconeTestStore()
+ {
+ }
+
+ protected override async Task StartAsync()
+ {
+ this._container = await this.CreateContainerAsync();
+
+ string restUrl = $"http://{this._container.Hostname}:{this._container.GetMappedPublicPort(RestPort)}";
+ string grpcUrl = $"http://{this._container.Hostname}:{this._container.GetMappedPublicPort(GrpcPort)}";
+
+ GrpcChannelOptions grpcOptions = new()
+ {
+ HttpClient = new()
+ {
+ BaseAddress = new(grpcUrl)
+ }
+ };
+
+ ClientOptions clientOptions = new()
+ {
+ BaseUrl = restUrl,
+ MaxRetries = 0,
+ IsTlsEnabled = false,
+ GrpcOptions = grpcOptions
+ };
+
+ this._client = new Pinecone.PineconeClient(
+ apiKey: "ForPineconeLocalTheApiKeysAreIgnored",
+ clientOptions: clientOptions);
+
+ Environment.SetEnvironmentVariable("PINECONE_GRPC_ENDPOINT", grpcUrl);
+
+ this._defaultVectorStore = new(this._client);
+ }
+
+ protected override async Task StopAsync()
+ {
+ if (this._container is not null)
+ {
+ await this._container.DisposeAsync();
+ }
+ }
+
+ private async Task CreateContainerAsync()
+ {
+ var container = new ContainerBuilder()
+ .WithImage(Image)
+ .WithPortBinding(RestPort, assignRandomHostPort: true)
+ .WithPortBinding(GrpcPort, assignRandomHostPort: true)
+ .Build();
+
+ await container.StartAsync();
+
+ return container;
+ }
+}
diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/VectorSearch/PineconeVectorSearchDistanceFunctionComplianceTests.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/VectorSearch/PineconeVectorSearchDistanceFunctionComplianceTests.cs
new file mode 100644
index 000000000000..f7dfc8e76834
--- /dev/null
+++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/VectorSearch/PineconeVectorSearchDistanceFunctionComplianceTests.cs
@@ -0,0 +1,26 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using PineconeIntegrationTests.Support;
+using VectorDataSpecificationTests.VectorSearch;
+using Xunit;
+
+namespace PineconeIntegrationTests.VectorSearch;
+
+public class PineconeVectorSearchDistanceFunctionComplianceTests(PineconeFixture fixture)
+ : VectorSearchDistanceFunctionComplianceTests(fixture), IClassFixture
+{
+ public override Task CosineDistance()
+ => Assert.ThrowsAsync(base.CosineDistance);
+
+ public override Task EuclideanDistance()
+ => Assert.ThrowsAsync(base.EuclideanDistance);
+
+ public override Task Hamming()
+ => Assert.ThrowsAsync(base.Hamming);
+
+ public override Task ManhattanDistance()
+ => Assert.ThrowsAsync(base.ManhattanDistance);
+
+ public override Task NegativeDotProductSimilarity()
+ => Assert.ThrowsAsync(base.NegativeDotProductSimilarity);
+}
diff --git a/dotnet/src/VectorDataIntegrationTests/PostgresIntegrationTests/CRUD/PostgresCollectionConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/PostgresIntegrationTests/CRUD/PostgresCollectionConformanceTests.cs
new file mode 100644
index 000000000000..14815cb55af0
--- /dev/null
+++ b/dotnet/src/VectorDataIntegrationTests/PostgresIntegrationTests/CRUD/PostgresCollectionConformanceTests.cs
@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using PostgresIntegrationTests.Support;
+using VectorDataSpecificationTests.CRUD;
+using Xunit;
+
+namespace PostgresIntegrationTests.CRUD;
+
+public class PostgresCollectionConformanceTests(PostgresFixture fixture)
+ : CollectionConformanceTests(fixture), IClassFixture
+{
+}
diff --git a/dotnet/src/VectorDataIntegrationTests/PostgresIntegrationTests/CRUD/PostgresRecordConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/PostgresIntegrationTests/CRUD/PostgresRecordConformanceTests.cs
new file mode 100644
index 000000000000..6f290168ead4
--- /dev/null
+++ b/dotnet/src/VectorDataIntegrationTests/PostgresIntegrationTests/CRUD/PostgresRecordConformanceTests.cs
@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using PostgresIntegrationTests.Support;
+using VectorDataSpecificationTests.CRUD;
+using Xunit;
+
+namespace PostgresIntegrationTests.CRUD;
+
+public class PostgresRecordConformanceTests(PostgresFixture fixture)
+ : RecordConformanceTests(fixture), IClassFixture
+{
+}
diff --git a/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisCollectionConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisCollectionConformanceTests.cs
new file mode 100644
index 000000000000..cc2ade48d481
--- /dev/null
+++ b/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisCollectionConformanceTests.cs
@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using RedisIntegrationTests.Support;
+using VectorDataSpecificationTests.CRUD;
+using Xunit;
+
+namespace RedisIntegrationTests.CRUD;
+
+public class RedisCollectionConformanceTests(RedisFixture fixture)
+ : CollectionConformanceTests(fixture), IClassFixture
+{
+}
diff --git a/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisGenericDataModelConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisGenericDataModelConformanceTests.cs
new file mode 100644
index 000000000000..b0c4f13084bd
--- /dev/null
+++ b/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisGenericDataModelConformanceTests.cs
@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using RedisIntegrationTests.Support;
+using VectorDataSpecificationTests.CRUD;
+using Xunit;
+
+namespace RedisIntegrationTests.CRUD;
+
+public class RedisGenericDataModelConformanceTests(RedisFixture fixture)
+ : GenericDataModelConformanceTests(fixture), IClassFixture
+{
+}
diff --git a/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisRecordConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisRecordConformanceTests.cs
new file mode 100644
index 000000000000..87b2d5a89e65
--- /dev/null
+++ b/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisRecordConformanceTests.cs
@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using RedisIntegrationTests.Support;
+using VectorDataSpecificationTests.CRUD;
+using Xunit;
+
+namespace RedisIntegrationTests.CRUD;
+
+public class RedisRecordConformanceTests(RedisFixture fixture)
+ : RecordConformanceTests(fixture), IClassFixture
+{
+}
diff --git a/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/Support/RedisFixture.cs b/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/Support/RedisFixture.cs
new file mode 100644
index 000000000000..a8c7004eaec2
--- /dev/null
+++ b/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/Support/RedisFixture.cs
@@ -0,0 +1,10 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using VectorDataSpecificationTests.Support;
+
+namespace RedisIntegrationTests.Support;
+
+public class RedisFixture : VectorStoreFixture
+{
+ public override TestStore TestStore => RedisTestStore.Instance;
+}
diff --git a/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerCollectionConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerCollectionConformanceTests.cs
new file mode 100644
index 000000000000..acbe977e8fa7
--- /dev/null
+++ b/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerCollectionConformanceTests.cs
@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using SqlServerIntegrationTests.Support;
+using VectorDataSpecificationTests.CRUD;
+using Xunit;
+
+namespace SqlServerIntegrationTests.CRUD;
+
+public class SqlServerCollectionConformanceTests(SqlServerFixture fixture)
+ : CollectionConformanceTests(fixture), IClassFixture
+{
+}
diff --git a/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerRecordConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerRecordConformanceTests.cs
new file mode 100644
index 000000000000..96939b19db23
--- /dev/null
+++ b/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerRecordConformanceTests.cs
@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using SqlServerIntegrationTests.Support;
+using VectorDataSpecificationTests.CRUD;
+using Xunit;
+
+namespace SqlServerIntegrationTests.CRUD;
+
+public class SqlServerRecordConformanceTests(SqlServerFixture fixture)
+ : RecordConformanceTests(fixture), IClassFixture
+{
+}
diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/BatchConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/BatchConformanceTests.cs
index ace837591a74..ab83e58c8082 100644
--- a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/BatchConformanceTests.cs
+++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/BatchConformanceTests.cs
@@ -66,4 +66,65 @@ await this.ExecuteAsync(async collection =>
Assert.Equal("keys", ex.ParamName);
});
}
+
+ [ConditionalFact]
+ public Task CanInsertUpdateAndDelete_WithVectors()
+ => this.CanInsertUpdateAndDelete(includeVectors: true);
+
+ [ConditionalFact]
+ public Task CanInsertUpdateAndDelete_WithoutVectors()
+ => this.CanInsertUpdateAndDelete(includeVectors: false);
+
+ private async Task CanInsertUpdateAndDelete(bool includeVectors)
+ {
+ await this.ExecuteAsync(async collection =>
+ {
+ SimpleModel[] inserted = Enumerable.Range(0, 10).Select(i => new SimpleModel()
+ {
+ Id = this.Fixture.GenerateNextKey(),
+ Number = 100 + i,
+ Text = i.ToString(),
+ Floats = Enumerable.Range(0, 10).Select(j => (float)(i + j)).ToArray()
+ }).ToArray();
+
+ TKey[] keys = await collection.UpsertBatchAsync(inserted).ToArrayAsync();
+ Assert.Equal(
+ inserted.Select(r => r.Id).OrderBy(id => id).ToArray(),
+ keys.OrderBy(id => id).ToArray());
+
+ SimpleModel[] received = await collection.GetBatchAsync(keys, new() { IncludeVectors = includeVectors }).ToArrayAsync();
+ for (int i = 0; i < inserted.Length; i++)
+ {
+ this.AssertEqual(inserted[i], this.GetRecord(received, inserted[i].Id!), includeVectors);
+ }
+
+ SimpleModel[] updated = inserted.Select(i => new SimpleModel()
+ {
+ Id = i.Id,
+ Text = i.Text + "updated",
+ Number = i.Number + 200,
+ Floats = i.Floats
+ }).ToArray();
+
+ keys = await collection.UpsertBatchAsync(updated).ToArrayAsync();
+ Assert.Equal(
+ updated.Select(r => r.Id).OrderBy(id => id).ToArray(),
+ keys.OrderBy(id => id).ToArray());
+
+ received = await collection.GetBatchAsync(keys, new() { IncludeVectors = includeVectors }).ToArrayAsync();
+ for (int i = 0; i < updated.Length; i++)
+ {
+ this.AssertEqual(updated[i], this.GetRecord(received, updated[i].Id!), includeVectors);
+ }
+
+ await collection.DeleteBatchAsync(keys);
+
+ Assert.False(await collection.GetBatchAsync(keys).AnyAsync());
+ });
+ }
+
+ // The order of records in the received array is not guaranteed
+ // to match the order of keys in the requested keys array.
+ private SimpleModel GetRecord(SimpleModel[] received, TKey key)
+ => received.Single(r => r.Id!.Equals(key));
}
diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/CollectionConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/CollectionConformanceTests.cs
new file mode 100644
index 000000000000..4e23b0c8a249
--- /dev/null
+++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/CollectionConformanceTests.cs
@@ -0,0 +1,56 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Extensions.VectorData;
+using VectorDataSpecificationTests.Models;
+using VectorDataSpecificationTests.Support;
+using VectorDataSpecificationTests.Xunit;
+using Xunit;
+
+namespace VectorDataSpecificationTests.CRUD;
+
+public class CollectionConformanceTests(VectorStoreFixture fixture)
+ : ConformanceTestsBase>(fixture) where TKey : notnull
+{
+ [ConditionalFact]
+ public async Task DeletingNonExistingCollectionDoesNotThrow()
+ {
+ await this.ExecuteAsync(async collection =>
+ {
+ Assert.False(await collection.CollectionExistsAsync());
+
+ await collection.DeleteCollectionAsync();
+ }, createCollection: false);
+ }
+
+ [ConditionalFact]
+ public async Task CreateCollectionIfNotExistsCalledMoreThanOnceDoesNotThrow()
+ {
+ await this.ExecuteAsync(async collection =>
+ {
+ Assert.False(await collection.CollectionExistsAsync());
+
+ await collection.CreateCollectionIfNotExistsAsync();
+ Assert.True(await collection.CollectionExistsAsync());
+ Assert.True(await this.Fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collection.CollectionName));
+
+ await collection.CreateCollectionIfNotExistsAsync();
+ }, createCollection: false);
+ }
+
+ [ConditionalFact]
+ public async Task CreateCollectionAsyncCalledMoreThanOnceThrows()
+ {
+ await this.ExecuteAsync(async collection =>
+ {
+ Assert.False(await collection.CollectionExistsAsync());
+
+ await collection.CreateCollectionAsync();
+ Assert.True(await collection.CollectionExistsAsync());
+ Assert.True(await this.Fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collection.CollectionName));
+
+ await collection.CreateCollectionIfNotExistsAsync();
+
+ await Assert.ThrowsAsync(() => collection.CreateCollectionAsync());
+ }, createCollection: false);
+ }
+}
diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/ConformanceTestsBase.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/ConformanceTestsBase.cs
index 21a6c95f8986..7dbaf04fbcba 100644
--- a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/ConformanceTestsBase.cs
+++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/ConformanceTestsBase.cs
@@ -1,7 +1,9 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.VectorData;
+using VectorDataSpecificationTests.Models;
using VectorDataSpecificationTests.Support;
+using Xunit;
namespace VectorDataSpecificationTests.CRUD;
@@ -14,13 +16,16 @@ public abstract class ConformanceTestsBase(VectorStoreFixture fix
protected virtual VectorStoreRecordDefinition? GetRecordDefinition() => null;
- protected async Task ExecuteAsync(Func, Task> test)
+ protected async Task ExecuteAsync(Func, Task> test, bool createCollection = true)
{
string collectionName = this.GetUniqueCollectionName();
var collection = this.Fixture.TestStore.DefaultVectorStore.GetCollection(collectionName,
this.GetRecordDefinition());
- await collection.CreateCollectionAsync();
+ if (createCollection)
+ {
+ await collection.CreateCollectionAsync();
+ }
try
{
@@ -31,4 +36,58 @@ protected async Task ExecuteAsync(Func))
+ {
+ var expectedSimple = (SimpleModel)(object)expected!;
+ var actualSimple = (SimpleModel)(object)actual!;
+
+ Assert.Equal(expectedSimple!.Id, actualSimple!.Id);
+ Assert.Equal(expectedSimple.Text, actualSimple.Text);
+ Assert.Equal(expectedSimple.Number, actualSimple.Number);
+
+ if (includeVectors)
+ {
+ Assert.Equal(expectedSimple.Floats.ToArray(), actualSimple.Floats.ToArray());
+ }
+ else
+ {
+ Assert.Equal(0, actualSimple.Floats.Length);
+ }
+ }
+ else if (typeof(TRecord) == typeof(VectorStoreGenericDataModel))
+ {
+ var expectedGeneric = (VectorStoreGenericDataModel)(object)expected!;
+ var actualGeneric = (VectorStoreGenericDataModel)(object)actual!;
+
+ Assert.Equal(expectedGeneric!.Key, actualGeneric!.Key);
+
+ foreach (var pair in expectedGeneric.Data)
+ {
+ Assert.Equal(pair.Value, actualGeneric.Data[pair.Key]);
+ }
+
+ foreach (var pair in expectedGeneric.Vectors)
+ {
+ if (includeVectors)
+ {
+ Assert.Equal(
+ ((ReadOnlyMemory)pair.Value!).ToArray(),
+ ((ReadOnlyMemory)actualGeneric.Vectors[pair.Key]!).ToArray());
+ }
+ else
+ {
+ Assert.Equal(0, ((ReadOnlyMemory)actualGeneric.Vectors[pair.Key]!).Length);
+ }
+ }
+ }
+ else
+ {
+ throw new NotSupportedException($"Type {typeof(TRecord)} is not supported.");
+ }
+ }
}
diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/GenericDataModelConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/GenericDataModelConformanceTests.cs
index 91ac166aafd4..2922b4d7108c 100644
--- a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/GenericDataModelConformanceTests.cs
+++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/GenericDataModelConformanceTests.cs
@@ -45,17 +45,7 @@ await this.ExecuteAsync(async collection =>
Assert.Equal(inserted.Key, key);
VectorStoreGenericDataModel? received = await collection.GetAsync(key, new() { IncludeVectors = true });
- Assert.NotNull(received);
-
- Assert.Equal(received.Key, key);
- foreach (var pair in inserted.Data)
- {
- Assert.Equal(pair.Value, received.Data[pair.Key]);
- }
-
- Assert.Equal(
- ((ReadOnlyMemory)inserted.Vectors[EmbeddingPropertyName]!).ToArray(),
- ((ReadOnlyMemory)received.Vectors[EmbeddingPropertyName]!).ToArray());
+ this.AssertEqual(inserted, received, includeVectors: true);
await collection.DeleteAsync(key);
diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/RecordConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/RecordConformanceTests.cs
new file mode 100644
index 000000000000..fb267239844f
--- /dev/null
+++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/RecordConformanceTests.cs
@@ -0,0 +1,72 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using VectorDataSpecificationTests.Models;
+using VectorDataSpecificationTests.Support;
+using VectorDataSpecificationTests.Xunit;
+using Xunit;
+
+namespace VectorDataSpecificationTests.CRUD;
+
+public class RecordConformanceTests(VectorStoreFixture fixture)
+ : ConformanceTestsBase>(fixture) where TKey : notnull
+{
+ [ConditionalFact]
+ public async Task ReadingAndDeletingNonExistingRecordDoesNotThrow()
+ {
+ await this.ExecuteAsync(async collection =>
+ {
+ TKey key = this.Fixture.GenerateNextKey();
+
+ Assert.Null(await collection.GetAsync(key));
+ await collection.DeleteAsync(key);
+ });
+ }
+
+ [ConditionalFact]
+ public Task CanInsertUpdateAndDelete_WithVectors()
+ => this.CanInsertUpdateAndDelete(includeVectors: true);
+
+ [ConditionalFact]
+ public Task CanInsertUpdateAndDelete_WithoutVectors()
+ => this.CanInsertUpdateAndDelete(includeVectors: false);
+
+ private async Task CanInsertUpdateAndDelete(bool includeVectors)
+ {
+ await this.ExecuteAsync(async collection =>
+ {
+ TKey expectedKey = this.Fixture.GenerateNextKey();
+ SimpleModel inserted = new()
+ {
+ Id = expectedKey,
+ Text = "some",
+ Number = 123,
+ Floats = new ReadOnlyMemory(Enumerable.Repeat(0.1f, SimpleModel.DimensionCount).ToArray())
+ };
+
+ TKey key = await collection.UpsertAsync(inserted);
+ Assert.Equal(expectedKey, key);
+
+ SimpleModel? received = await collection.GetAsync(expectedKey, new() { IncludeVectors = includeVectors });
+ this.AssertEqual(inserted, received, includeVectors);
+
+ SimpleModel updated = new()
+ {
+ Id = expectedKey,
+ Text = "updated",
+ Number = 456,
+ Floats = new ReadOnlyMemory(Enumerable.Repeat(0.2f, SimpleModel.DimensionCount).ToArray())
+ };
+
+ key = await collection.UpsertAsync(updated);
+ Assert.Equal(expectedKey, key);
+
+ received = await collection.GetAsync(expectedKey, new() { IncludeVectors = includeVectors });
+ this.AssertEqual(updated, received, includeVectors);
+
+ await collection.DeleteAsync(key);
+
+ received = await collection.GetAsync(key);
+ Assert.Null(received);
+ });
+ }
+}
diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Models/SimpleModel.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Models/SimpleModel.cs
index 0646f0fe2f1f..3f99d6186f94 100644
--- a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Models/SimpleModel.cs
+++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Models/SimpleModel.cs
@@ -11,6 +11,8 @@ namespace VectorDataSpecificationTests.Models;
/// TKey is a generic parameter because different connectors support different key types.
public sealed class SimpleModel
{
+ public const int DimensionCount = 10;
+
[VectorStoreRecordKey(StoragePropertyName = "key")]
public TKey? Id { get; set; }
@@ -20,6 +22,6 @@ public sealed class SimpleModel
[VectorStoreRecordData(StoragePropertyName = "number")]
public int Number { get; set; }
- [VectorStoreRecordVector(Dimensions: 10, StoragePropertyName = "embedding")]
+ [VectorStoreRecordVector(Dimensions: DimensionCount, StoragePropertyName = "embedding")]
public ReadOnlyMemory Floats { get; set; }
}