From f856c7fd312a877513eb36d3805cdc9444eeb007 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 13 Aug 2024 16:30:44 +0100 Subject: [PATCH 1/2] Embeddings update + OllamaSharp latest --- dotnet/Directory.Packages.props | 8 +++--- .../OllamaTextEmbeddingGenerationTests.cs | 9 +++--- .../TestData/embeddings_test_response.json | 28 +++++++++++-------- .../OllamaTextEmbeddingGenerationService.cs | 25 +++++++++-------- .../Ollama/OllamaCompletionTests.cs | 12 ++++---- .../Ollama/OllamaTextEmbeddingTests.cs | 4 +-- .../Ollama/OllamaTextGenerationTests.cs | 12 ++++---- 7 files changed, 53 insertions(+), 45 deletions(-) diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index b76695dcb047..74f5fab75658 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -38,7 +38,7 @@ - + @@ -135,8 +135,8 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - - - + + + \ No newline at end of file diff --git a/dotnet/src/Connectors/Connectors.Ollama.UnitTests/Services/OllamaTextEmbeddingGenerationTests.cs b/dotnet/src/Connectors/Connectors.Ollama.UnitTests/Services/OllamaTextEmbeddingGenerationTests.cs index 4e9cf00754cf..53462080eb06 100644 --- a/dotnet/src/Connectors/Connectors.Ollama.UnitTests/Services/OllamaTextEmbeddingGenerationTests.cs +++ b/dotnet/src/Connectors/Connectors.Ollama.UnitTests/Services/OllamaTextEmbeddingGenerationTests.cs @@ -68,12 +68,12 @@ public async Task ShouldSendPromptToServiceAsync() httpClient: this._httpClient); //Act - await sut.GenerateEmbeddingsAsync(new List { "fake-text" }); + await sut.GenerateEmbeddingsAsync(["fake-text"]); //Assert var requestPayload = JsonSerializer.Deserialize(this._messageHandlerStub.RequestContent); Assert.NotNull(requestPayload); - Assert.Equal("fake-text", requestPayload.Prompt); + Assert.Equal("fake-text", requestPayload.Input[0]); } [Fact] @@ -90,9 +90,10 @@ public async Task ShouldHandleServiceResponseAsync() //Assert Assert.NotNull(contents); + Assert.Equal(2, contents.Count); - var content = contents.SingleOrDefault(); - Assert.Equal(8, content.Length); + var content = contents[0]; + Assert.Equal(5, content.Length); } public void Dispose() diff --git a/dotnet/src/Connectors/Connectors.Ollama.UnitTests/TestData/embeddings_test_response.json b/dotnet/src/Connectors/Connectors.Ollama.UnitTests/TestData/embeddings_test_response.json index 63218204d30d..3316addba6dd 100644 --- a/dotnet/src/Connectors/Connectors.Ollama.UnitTests/TestData/embeddings_test_response.json +++ b/dotnet/src/Connectors/Connectors.Ollama.UnitTests/TestData/embeddings_test_response.json @@ -1,13 +1,19 @@ { - "embedding": [ - -0.08541165292263031, - 0.08639130741357803, - -0.12805694341659546, - -0.2877824902534485, - 0.2114177942276001, - -0.29374566674232483, - -0.10496602207422256, - 0.009402364492416382 - ], - "model": "fake-model" + "model": "fake-model", + "embeddings": [ + [ + 0.020765934, + 0.007495159, + 0.01268963, + 0.013938076, + -0.04621073 + ], + [ + 0.025005031, + 0.009804744, + -0.016960088, + -0.024823941, + -0.02756831 + ] + ] } \ No newline at end of file diff --git a/dotnet/src/Connectors/Connectors.Ollama/Services/OllamaTextEmbeddingGenerationService.cs b/dotnet/src/Connectors/Connectors.Ollama/Services/OllamaTextEmbeddingGenerationService.cs index 13adcd165c80..9e152f917f88 100644 --- a/dotnet/src/Connectors/Connectors.Ollama/Services/OllamaTextEmbeddingGenerationService.cs +++ b/dotnet/src/Connectors/Connectors.Ollama/Services/OllamaTextEmbeddingGenerationService.cs @@ -9,6 +9,7 @@ using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel.Connectors.Ollama.Core; using Microsoft.SemanticKernel.Embeddings; +using Microsoft.SemanticKernel.Services; using OllamaSharp; using OllamaSharp.Models; @@ -58,20 +59,20 @@ public async Task>> GenerateEmbeddingsAsync( Kernel? kernel = null, CancellationToken cancellationToken = default) { - var tasks = new List>(); - foreach (var prompt in data) + var request = new GenerateEmbeddingRequest { - tasks.Add(this._client.GenerateEmbeddings(prompt, cancellationToken: cancellationToken)); - } + Model = this.GetModelId()!, + Input = data.ToList() + }; + + var response = await this._client.GenerateEmbeddings(request, cancellationToken: cancellationToken).ConfigureAwait(false); - await Task.WhenAll(tasks.ToArray()).ConfigureAwait(false); + List> embeddings = []; + foreach (var embedding in response.Embeddings) + { + embeddings.Add(embedding.Select(@decimal => (float)@decimal).ToArray()); + } - return new List>( - tasks.Select( - task => new ReadOnlyMemory(task.Result.Embedding - .Select(@decimal => (float)@decimal).ToArray() - ) - ) - ); + return embeddings; } } diff --git a/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaCompletionTests.cs b/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaCompletionTests.cs index 4fabf80936ff..3d6f5bc990f6 100644 --- a/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaCompletionTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaCompletionTests.cs @@ -29,7 +29,7 @@ public sealed class OllamaCompletionTests(ITestOutputHelper output) : IDisposabl .AddUserSecrets() .Build(); - [Theory(Skip = "For manual verification only")] + [Theory]//(Skip = "For manual verification only")] [InlineData("Where is the most famous fish market in Seattle, Washington, USA?", "Pike Place")] public async Task ItInvokeStreamingWorksAsync(string prompt, string expectedAnswerContains) { @@ -59,7 +59,7 @@ public async Task ItInvokeStreamingWorksAsync(string prompt, string expectedAnsw Assert.Contains(expectedAnswerContains, fullResult.ToString(), StringComparison.OrdinalIgnoreCase); } - [Fact(Skip = "For manual verification only")] + [Fact]//(Skip = "For manual verification only")] public async Task ItShouldReturnMetadataAsync() { // Arrange @@ -86,7 +86,7 @@ public async Task ItShouldReturnMetadataAsync() Assert.True(lastUpdate.Metadata.TryGetValue("CreatedAt", out object? createdAt)); } - [Theory(Skip = "For manual verification only")] + [Theory]//(Skip = "For manual verification only")] [InlineData("\n")] [InlineData("\r\n")] public async Task ItCompletesWithDifferentLineEndingsAsync(string lineEnding) @@ -113,7 +113,7 @@ public async Task ItCompletesWithDifferentLineEndingsAsync(string lineEnding) Assert.Contains(ExpectedAnswerContains, actual.GetValue(), StringComparison.OrdinalIgnoreCase); } - [Fact(Skip = "For manual verification only")] + [Fact]//(Skip = "For manual verification only")] public async Task ItInvokePromptTestAsync() { // Arrange @@ -131,7 +131,7 @@ public async Task ItInvokePromptTestAsync() Assert.Contains("Pike Place", actual.GetValue(), StringComparison.OrdinalIgnoreCase); } - [Theory(Skip = "For manual verification only")] + [Theory]//(Skip = "For manual verification only")] [InlineData("Where is the most famous fish market in Seattle, Washington, USA?", "Pike Place")] public async Task ItInvokeTestAsync(string prompt, string expectedAnswerContains) { @@ -152,7 +152,7 @@ public async Task ItInvokeTestAsync(string prompt, string expectedAnswerContains Assert.Contains(expectedAnswerContains, actual.GetValue(), StringComparison.OrdinalIgnoreCase); } - [Fact(Skip = "For manual verification only")] + [Fact]//(Skip = "For manual verification only")] public async Task ItShouldHaveSemanticKernelVersionHeaderAsync() { // Arrange diff --git a/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaTextEmbeddingTests.cs b/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaTextEmbeddingTests.cs index f530098b473b..9311f7987468 100644 --- a/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaTextEmbeddingTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaTextEmbeddingTests.cs @@ -18,7 +18,7 @@ public sealed class OllamaTextEmbeddingTests .AddUserSecrets() .Build(); - [Theory(Skip = "For manual verification only")] + [Theory]//(Skip = "For manual verification only")] [InlineData("mxbai-embed-large", 1024)] [InlineData("nomic-embed-text", 768)] [InlineData("all-minilm", 384)] @@ -42,7 +42,7 @@ public async Task GenerateEmbeddingHasExpectedLengthForModelAsync(string modelId Assert.Equal(expectedVectorLength, result.Length); } - [Theory(Skip = "For manual verification only")] + [Theory]//(Skip = "For manual verification only")] [InlineData("mxbai-embed-large", 1024)] [InlineData("nomic-embed-text", 768)] [InlineData("all-minilm", 384)] diff --git a/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaTextGenerationTests.cs b/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaTextGenerationTests.cs index 597fdf331db2..a104fa4000b4 100644 --- a/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaTextGenerationTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaTextGenerationTests.cs @@ -29,7 +29,7 @@ public sealed class OllamaTextGenerationTests(ITestOutputHelper output) : IDispo .AddUserSecrets() .Build(); - [Theory(Skip = "For manual verification only")] + [Theory]//(Skip = "For manual verification only")] [InlineData("Where is the most famous fish market in Seattle, Washington, USA?", "Pike Place")] public async Task ItInvokeStreamingWorksAsync(string prompt, string expectedAnswerContains) { @@ -55,7 +55,7 @@ public async Task ItInvokeStreamingWorksAsync(string prompt, string expectedAnsw Assert.Contains(expectedAnswerContains, fullResult.ToString(), StringComparison.OrdinalIgnoreCase); } - [Fact(Skip = "For manual verification only")] + [Fact]//(Skip = "For manual verification only")] public async Task ItShouldReturnMetadataAsync() { // Arrange @@ -87,7 +87,7 @@ public async Task ItShouldReturnMetadataAsync() Assert.NotEqual(0, ollamaMetadata.EvalDuration); } - [Theory(Skip = "For manual verification only")] + [Theory]//(Skip = "For manual verification only")] [InlineData("\n")] [InlineData("\r\n")] public async Task ItCompletesWithDifferentLineEndingsAsync(string lineEnding) @@ -114,7 +114,7 @@ public async Task ItCompletesWithDifferentLineEndingsAsync(string lineEnding) Assert.Contains(ExpectedAnswerContains, actual.GetValue(), StringComparison.OrdinalIgnoreCase); } - [Fact(Skip = "For manual verification only")] + [Fact]//(Skip = "For manual verification only")] public async Task ItInvokePromptTestAsync() { // Arrange @@ -132,7 +132,7 @@ public async Task ItInvokePromptTestAsync() Assert.Contains("Pike Place", actual.GetValue(), StringComparison.OrdinalIgnoreCase); } - [Theory(Skip = "For manual verification only")] + [Theory]//(Skip = "For manual verification only")] [InlineData("Where is the most famous fish market in Seattle, Washington, USA?", "Pike Place")] public async Task ItInvokeTestAsync(string prompt, string expectedAnswerContains) { @@ -154,7 +154,7 @@ public async Task ItInvokeTestAsync(string prompt, string expectedAnswerContains Assert.NotNull(actual.Metadata); } - [Fact(Skip = "For manual verification only")] + [Fact]//(Skip = "For manual verification only")] public async Task ItShouldHaveSemanticKernelVersionHeaderAsync() { // Arrange From 80b3f086b69639801a3d7ce90144e0e8ee942fab Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 13 Aug 2024 16:37:15 +0100 Subject: [PATCH 2/2] Update IT to be manual --- .../Connectors/Ollama/OllamaCompletionTests.cs | 12 ++++++------ .../Connectors/Ollama/OllamaTextEmbeddingTests.cs | 4 ++-- .../Connectors/Ollama/OllamaTextGenerationTests.cs | 12 ++++++------ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaCompletionTests.cs b/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaCompletionTests.cs index 3d6f5bc990f6..4fabf80936ff 100644 --- a/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaCompletionTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaCompletionTests.cs @@ -29,7 +29,7 @@ public sealed class OllamaCompletionTests(ITestOutputHelper output) : IDisposabl .AddUserSecrets() .Build(); - [Theory]//(Skip = "For manual verification only")] + [Theory(Skip = "For manual verification only")] [InlineData("Where is the most famous fish market in Seattle, Washington, USA?", "Pike Place")] public async Task ItInvokeStreamingWorksAsync(string prompt, string expectedAnswerContains) { @@ -59,7 +59,7 @@ public async Task ItInvokeStreamingWorksAsync(string prompt, string expectedAnsw Assert.Contains(expectedAnswerContains, fullResult.ToString(), StringComparison.OrdinalIgnoreCase); } - [Fact]//(Skip = "For manual verification only")] + [Fact(Skip = "For manual verification only")] public async Task ItShouldReturnMetadataAsync() { // Arrange @@ -86,7 +86,7 @@ public async Task ItShouldReturnMetadataAsync() Assert.True(lastUpdate.Metadata.TryGetValue("CreatedAt", out object? createdAt)); } - [Theory]//(Skip = "For manual verification only")] + [Theory(Skip = "For manual verification only")] [InlineData("\n")] [InlineData("\r\n")] public async Task ItCompletesWithDifferentLineEndingsAsync(string lineEnding) @@ -113,7 +113,7 @@ public async Task ItCompletesWithDifferentLineEndingsAsync(string lineEnding) Assert.Contains(ExpectedAnswerContains, actual.GetValue(), StringComparison.OrdinalIgnoreCase); } - [Fact]//(Skip = "For manual verification only")] + [Fact(Skip = "For manual verification only")] public async Task ItInvokePromptTestAsync() { // Arrange @@ -131,7 +131,7 @@ public async Task ItInvokePromptTestAsync() Assert.Contains("Pike Place", actual.GetValue(), StringComparison.OrdinalIgnoreCase); } - [Theory]//(Skip = "For manual verification only")] + [Theory(Skip = "For manual verification only")] [InlineData("Where is the most famous fish market in Seattle, Washington, USA?", "Pike Place")] public async Task ItInvokeTestAsync(string prompt, string expectedAnswerContains) { @@ -152,7 +152,7 @@ public async Task ItInvokeTestAsync(string prompt, string expectedAnswerContains Assert.Contains(expectedAnswerContains, actual.GetValue(), StringComparison.OrdinalIgnoreCase); } - [Fact]//(Skip = "For manual verification only")] + [Fact(Skip = "For manual verification only")] public async Task ItShouldHaveSemanticKernelVersionHeaderAsync() { // Arrange diff --git a/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaTextEmbeddingTests.cs b/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaTextEmbeddingTests.cs index 9311f7987468..f530098b473b 100644 --- a/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaTextEmbeddingTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaTextEmbeddingTests.cs @@ -18,7 +18,7 @@ public sealed class OllamaTextEmbeddingTests .AddUserSecrets() .Build(); - [Theory]//(Skip = "For manual verification only")] + [Theory(Skip = "For manual verification only")] [InlineData("mxbai-embed-large", 1024)] [InlineData("nomic-embed-text", 768)] [InlineData("all-minilm", 384)] @@ -42,7 +42,7 @@ public async Task GenerateEmbeddingHasExpectedLengthForModelAsync(string modelId Assert.Equal(expectedVectorLength, result.Length); } - [Theory]//(Skip = "For manual verification only")] + [Theory(Skip = "For manual verification only")] [InlineData("mxbai-embed-large", 1024)] [InlineData("nomic-embed-text", 768)] [InlineData("all-minilm", 384)] diff --git a/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaTextGenerationTests.cs b/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaTextGenerationTests.cs index a104fa4000b4..597fdf331db2 100644 --- a/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaTextGenerationTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Ollama/OllamaTextGenerationTests.cs @@ -29,7 +29,7 @@ public sealed class OllamaTextGenerationTests(ITestOutputHelper output) : IDispo .AddUserSecrets() .Build(); - [Theory]//(Skip = "For manual verification only")] + [Theory(Skip = "For manual verification only")] [InlineData("Where is the most famous fish market in Seattle, Washington, USA?", "Pike Place")] public async Task ItInvokeStreamingWorksAsync(string prompt, string expectedAnswerContains) { @@ -55,7 +55,7 @@ public async Task ItInvokeStreamingWorksAsync(string prompt, string expectedAnsw Assert.Contains(expectedAnswerContains, fullResult.ToString(), StringComparison.OrdinalIgnoreCase); } - [Fact]//(Skip = "For manual verification only")] + [Fact(Skip = "For manual verification only")] public async Task ItShouldReturnMetadataAsync() { // Arrange @@ -87,7 +87,7 @@ public async Task ItShouldReturnMetadataAsync() Assert.NotEqual(0, ollamaMetadata.EvalDuration); } - [Theory]//(Skip = "For manual verification only")] + [Theory(Skip = "For manual verification only")] [InlineData("\n")] [InlineData("\r\n")] public async Task ItCompletesWithDifferentLineEndingsAsync(string lineEnding) @@ -114,7 +114,7 @@ public async Task ItCompletesWithDifferentLineEndingsAsync(string lineEnding) Assert.Contains(ExpectedAnswerContains, actual.GetValue(), StringComparison.OrdinalIgnoreCase); } - [Fact]//(Skip = "For manual verification only")] + [Fact(Skip = "For manual verification only")] public async Task ItInvokePromptTestAsync() { // Arrange @@ -132,7 +132,7 @@ public async Task ItInvokePromptTestAsync() Assert.Contains("Pike Place", actual.GetValue(), StringComparison.OrdinalIgnoreCase); } - [Theory]//(Skip = "For manual verification only")] + [Theory(Skip = "For manual verification only")] [InlineData("Where is the most famous fish market in Seattle, Washington, USA?", "Pike Place")] public async Task ItInvokeTestAsync(string prompt, string expectedAnswerContains) { @@ -154,7 +154,7 @@ public async Task ItInvokeTestAsync(string prompt, string expectedAnswerContains Assert.NotNull(actual.Metadata); } - [Fact]//(Skip = "For manual verification only")] + [Fact(Skip = "For manual verification only")] public async Task ItShouldHaveSemanticKernelVersionHeaderAsync() { // Arrange