diff --git a/.vscode/tasks.json b/.vscode/tasks.json index be95901abec2..c8b65d365445 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -74,10 +74,10 @@ "detail": "Runs tasks to validate changes before checking in.", "group": "test", "dependsOn": [ + "R# cleanup", "Build - Semantic-Kernel", "Test - Semantic-Kernel", - "Run - Kernel-Demo", - "R# cleanup" + "Run - Kernel-Demo" ], "dependsOrder": "sequence" }, @@ -103,7 +103,7 @@ "label": "Test - Semantic-Kernel", "command": "dotnet", "type": "process", - "args": ["test", "SemanticKernel.Test.csproj"], + "args": ["test", "SemanticKernel.UnitTests.csproj"], "problemMatcher": "$msCompile", "group": "test", "presentation": { @@ -112,7 +112,7 @@ "group": "PR-Validate" }, "options": { - "cwd": "${workspaceFolder}/dotnet/src/SemanticKernel.Test/" + "cwd": "${workspaceFolder}/dotnet/src/SemanticKernel.UnitTests/" } }, { @@ -123,7 +123,7 @@ "test", "--collect", "XPlat Code Coverage;Format=lcov", - "SemanticKernel.Test.csproj" + "SemanticKernel.UnitTests.csproj" ], "problemMatcher": "$msCompile", "group": "test", @@ -132,7 +132,7 @@ "panel": "shared" }, "options": { - "cwd": "${workspaceFolder}/dotnet/src/SemanticKernel.Test/" + "cwd": "${workspaceFolder}/dotnet/src/SemanticKernel.UnitTests/" } }, { diff --git a/dotnet/src/SemanticKernel.IntegrationTests/AI/OpenAICompletionTests.cs b/dotnet/src/SemanticKernel.IntegrationTests/AI/OpenAICompletionTests.cs index 524602496a85..0ed855347140 100644 --- a/dotnet/src/SemanticKernel.IntegrationTests/AI/OpenAICompletionTests.cs +++ b/dotnet/src/SemanticKernel.IntegrationTests/AI/OpenAICompletionTests.cs @@ -9,6 +9,7 @@ using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.KernelExtensions; using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Reliability; using SemanticKernel.IntegrationTests.TestSettings; using Xunit; using Xunit.Abstractions; @@ -91,6 +92,34 @@ public async Task AzureOpenAITestAsync(string prompt, string expectedAnswerConta Assert.Contains(expectedAnswerContains, actual.Result, StringComparison.InvariantCultureIgnoreCase); } + [Theory] + [InlineData("Where is the most famous fish market in Seattle, Washington, USA?", + "Error executing action [attempt 1 of 1]. Reason: Unauthorized. Will retry after 2000ms")] + public async Task OpenAIHttpRetryPolicyTestAsync(string prompt, string expectedOutput) + { + // Arrange + var retryConfig = new HttpRetryConfig(); + retryConfig.RetryableStatusCodes.Add(System.Net.HttpStatusCode.Unauthorized); + IKernel target = Kernel.Builder.WithLogger(this._testOutputHelper).Configure(c => c.SetDefaultHttpRetryConfig(retryConfig)).Build(); + + OpenAIConfiguration? openAIConfiguration = this._configuration.GetSection("OpenAI").Get(); + Assert.NotNull(openAIConfiguration); + + // Use an invalid API key to force a 401 Unauthorized response + target.Config.AddOpenAICompletionBackend( + label: openAIConfiguration.Label, + modelId: openAIConfiguration.ModelId, + apiKey: "INVALID_KEY"); + + IDictionary skill = GetSkill("SummarizeSkill", target); + + // Act + await target.RunAsync(prompt, skill["Summarize"]); + + // Assert + Assert.Contains(expectedOutput, this._testOutputHelper.GetLogs(), StringComparison.InvariantCultureIgnoreCase); + } + private static IDictionary GetSkill(string skillName, IKernel target) { string? currentAssemblyDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); diff --git a/dotnet/src/SemanticKernel.IntegrationTests/RedirectOutput.cs b/dotnet/src/SemanticKernel.IntegrationTests/RedirectOutput.cs index 4805d94dcf71..5aa902e6c9a2 100644 --- a/dotnet/src/SemanticKernel.IntegrationTests/RedirectOutput.cs +++ b/dotnet/src/SemanticKernel.IntegrationTests/RedirectOutput.cs @@ -1,18 +1,22 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.IO; using System.Text; +using Microsoft.Extensions.Logging; using Xunit.Abstractions; namespace SemanticKernel.IntegrationTests; -public class RedirectOutput : TextWriter +public class RedirectOutput : TextWriter, ILogger { private readonly ITestOutputHelper _output; + private readonly StringBuilder _logs; public RedirectOutput(ITestOutputHelper output) { this._output = output; + this._logs = new StringBuilder(); } public override Encoding Encoding { get; } = Encoding.UTF8; @@ -20,5 +24,28 @@ public RedirectOutput(ITestOutputHelper output) public override void WriteLine(string? value) { this._output.WriteLine(value); + this._logs.AppendLine(value); + } + + public IDisposable? BeginScope(TState state) where TState : notnull + { + return null; + } + + public bool IsEnabled(LogLevel logLevel) + { + return true; + } + + public string GetLogs() + { + return this._logs.ToString(); + } + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + var message = formatter(state, exception); + this._output?.WriteLine(message); + this._logs.AppendLine(message); } } diff --git a/dotnet/src/SemanticKernel.UnitTests/Configuration/KernelConfigTests.cs b/dotnet/src/SemanticKernel.UnitTests/Configuration/KernelConfigTests.cs index 43d94bfbc2cb..a8baf80b3675 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Configuration/KernelConfigTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Configuration/KernelConfigTests.cs @@ -16,55 +16,55 @@ namespace SemanticKernel.UnitTests.Configuration; public class KernelConfigTests { [Fact] - public void RetryMechanismIsSet() + public void HttpRetryHandlerFactoryIsSet() { // Arrange - var retry = new PassThroughWithoutRetry(); + var retry = new NullHttpRetryHandlerFactory(); var config = new KernelConfig(); // Act - config.SetRetryMechanism(retry); + config.SetHttpRetryHandlerFactory(retry); // Assert - Assert.Equal(retry, config.RetryMechanism); + Assert.Equal(retry, config.HttpHandlerFactory); } [Fact] - public void RetryMechanismIsSetWithCustomImplementation() + public void HttpRetryHandlerFactoryIsSetWithCustomImplementation() { // Arrange - var retry = new Mock(); + var retry = new Mock(); var config = new KernelConfig(); // Act - config.SetRetryMechanism(retry.Object); + config.SetHttpRetryHandlerFactory(retry.Object); // Assert - Assert.Equal(retry.Object, config.RetryMechanism); + Assert.Equal(retry.Object, config.HttpHandlerFactory); } [Fact] - public void RetryMechanismIsSetToPassThroughWithoutRetryIfNull() + public void HttpRetryHandlerFactoryIsSetToDefaultHttpRetryHandlerFactoryIfNull() { // Arrange var config = new KernelConfig(); // Act - config.SetRetryMechanism(null); + config.SetHttpRetryHandlerFactory(null); // Assert - Assert.IsType(config.RetryMechanism); + Assert.IsType(config.HttpHandlerFactory); } [Fact] - public void RetryMechanismIsSetToPassThroughWithoutRetryIfNotSet() + public void HttpRetryHandlerFactoryIsSetToDefaultHttpRetryHandlerFactoryIfNotSet() { // Arrange var config = new KernelConfig(); // Act // Assert - Assert.IsType(config.RetryMechanism); + Assert.IsType(config.HttpHandlerFactory); } [Fact] diff --git a/dotnet/src/SemanticKernel.UnitTests/Reliability/DefaultHttpRetryHandlerTests.cs b/dotnet/src/SemanticKernel.UnitTests/Reliability/DefaultHttpRetryHandlerTests.cs new file mode 100644 index 000000000000..fb5c62120de3 --- /dev/null +++ b/dotnet/src/SemanticKernel.UnitTests/Reliability/DefaultHttpRetryHandlerTests.cs @@ -0,0 +1,675 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Reliability; +using Moq; +using Moq.Protected; +using Xunit; + +namespace SemanticKernel.UnitTests.Reliability; + +public class DefaultHttpRetryHandlerTests +{ + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + [InlineData(HttpStatusCode.TooManyRequests)] + public async Task NoMaxRetryCountCallsOnceForStatusAsync(HttpStatusCode statusCode) + { + // Arrange + using var retry = new DefaultHttpRetryHandler(new HttpRetryConfig() { MaxRetryCount = 0 }, Mock.Of()); + using var mockResponse = new HttpResponseMessage(statusCode); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + [InlineData(HttpStatusCode.TooManyRequests)] + public async Task ItRetriesOnceOnRetryableStatusAsync(HttpStatusCode statusCode) + { + // Arrange + using var retry = ConfigureRetryHandler(); + using var mockResponse = new HttpResponseMessage(statusCode); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + } + + [Theory] + [InlineData(typeof(HttpRequestException))] + public async Task ItRetriesOnceOnRetryableExceptionAsync(Type exceptionType) + { + // Arrange + using var retry = ConfigureRetryHandler(); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(exceptionType); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await Assert.ThrowsAsync(exceptionType, + async () => await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Theory] + [InlineData(typeof(HttpRequestException))] + public async Task NoMaxRetryCountCallsOnceForExceptionAsync(Type exceptionType) + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 0 }); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(exceptionType); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await Assert.ThrowsAsync(exceptionType, + async () => await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + [InlineData(HttpStatusCode.TooManyRequests)] + public async Task ItRetriesOnceOnTransientStatusWithExponentialBackoffAsync(HttpStatusCode statusCode) + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { UseExponentialBackoff = true }); + using var mockResponse = new HttpResponseMessage(statusCode); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + } + + [Theory] + [InlineData(typeof(HttpRequestException))] + public async Task ItRetriesOnceOnRetryableExceptionWithExponentialBackoffAsync(Type exceptionType) + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { UseExponentialBackoff = true }); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(exceptionType); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await Assert.ThrowsAsync(exceptionType, + async () => await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + public async Task ItRetriesExponentiallyWithExponentialBackoffAsync(HttpStatusCode statusCode) + { + // Arrange + var currentTime = DateTimeOffset.UtcNow; + var mockTimeProvider = new Mock(); + var mockDelayProvider = new Mock(); + mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) + .Returns(() => currentTime) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(5)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(510)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(1015)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(1520)); + using var retry = ConfigureRetryHandler(new HttpRetryConfig() + { + UseExponentialBackoff = true, MaxRetryCount = 3, + MinRetryDelay = TimeSpan.FromMilliseconds(500) + }, mockTimeProvider, mockDelayProvider); + using var mockResponse = new HttpResponseMessage(statusCode); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(4), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(4)); + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(500), It.IsAny()), Times.Once); + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(1000), It.IsAny()), Times.Once); + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(2000), It.IsAny()), Times.Once); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + public async Task ItRetriesOnceOnTransientStatusCodeWithRetryValueAsync(HttpStatusCode statusCode) + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig(), null); + using var mockResponse = new HttpResponseMessage() + { + StatusCode = statusCode, + Headers = { RetryAfter = new RetryConditionHeaderValue(new TimeSpan(0, 0, 0, 1)) }, + }; + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + using var testContent = new StringContent("test"); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + Assert.Equal(new TimeSpan(0, 0, 0, 1), response.Headers.RetryAfter?.Delta); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + public async Task ItRetriesStatusCustomCountAsync(HttpStatusCode expectedStatus) + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 3 }, null); + using var mockResponse = new HttpResponseMessage(expectedStatus); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(4), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(expectedStatus, response.StatusCode); + } + + [Theory] + [InlineData(typeof(HttpRequestException))] + public async Task ItRetriesExceptionsCustomCountAsync(Type expectedException) + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 3 }, null); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(expectedException); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await Assert.ThrowsAsync(expectedException, + async () => await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(4), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Fact] + public async Task NoExceptionNoRetryAsync() + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 3 }, null); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.OK); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(1), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + } + + [Fact] + public async Task ItDoesNotExecuteOnCancellationTokenAsync() + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 3 }, null); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.OK); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + var cancellationToken = new CancellationToken(true); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await Assert.ThrowsAsync(async () => + await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, cancellationToken)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Never(), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Fact] + public async Task ItDoestExecuteOnFalseCancellationTokenAsync() + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 3 }, null); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.OK); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + var cancellationToken = new CancellationToken(false); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, cancellationToken); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(1), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + } + + [Fact] + public async Task ItRetriesWithMinRetryDelayAsync() + { + var HttpRetryConfig = new HttpRetryConfig + { + MinRetryDelay = TimeSpan.FromMilliseconds(500) + }; + + var mockDelayProvider = new Mock(); + var mockTimeProvider = new Mock(); + + var currentTime = DateTimeOffset.UtcNow; + + mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) + .Returns(() => currentTime) + .Returns(() => currentTime.AddMilliseconds(5)) + .Returns(() => currentTime.AddMilliseconds(510)); + + mockDelayProvider.Setup(x => x.DelayAsync(It.IsAny(), It.IsAny())) + .Returns(() => Task.CompletedTask); + + using var retry = ConfigureRetryHandler(HttpRetryConfig, mockTimeProvider, mockDelayProvider); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(2)); + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(500), It.IsAny()), Times.Once); + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + } + + [Fact] + public async Task ItRetriesWithMaxRetryDelayAsync() + { + var HttpRetryConfig = new HttpRetryConfig + { + MinRetryDelay = TimeSpan.FromMilliseconds(1), + MaxRetryDelay = TimeSpan.FromMilliseconds(500) + }; + + var mockDelayProvider = new Mock(); + var mockTimeProvider = new Mock(); + + var currentTime = DateTimeOffset.UtcNow; + + mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) + .Returns(() => currentTime) + .Returns(() => currentTime.AddMilliseconds(5)) + .Returns(() => currentTime.AddMilliseconds(505)); + + mockDelayProvider.Setup(x => x.DelayAsync(It.IsAny(), It.IsAny())) + .Returns(() => Task.CompletedTask); + + using var retry = ConfigureRetryHandler(HttpRetryConfig, mockTimeProvider, mockDelayProvider); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests) + { Headers = { RetryAfter = new RetryConditionHeaderValue(TimeSpan.FromMilliseconds(2000)) } }; + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(2)); + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(500), It.IsAny()), Times.Once); + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + Assert.Equal(TimeSpan.FromMilliseconds(2000), response.Headers.RetryAfter?.Delta); + } + + [Theory] + [InlineData(HttpStatusCode.TooManyRequests)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + [InlineData(HttpStatusCode.RequestTimeout)] + public async Task ItRetriesWithMaxTotalDelayAsync(HttpStatusCode statusCode) + { + // Arrange + var HttpRetryConfig = new HttpRetryConfig + { + MaxRetryCount = 5, + MinRetryDelay = TimeSpan.FromMilliseconds(50), + MaxRetryDelay = TimeSpan.FromMilliseconds(50), + MaxTotalRetryTime = TimeSpan.FromMilliseconds(350) + }; + + var mockDelayProvider = new Mock(); + var mockTimeProvider = new Mock(); + + var currentTime = DateTimeOffset.UtcNow; + mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) + .Returns(() => currentTime) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(5)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(55)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(110)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(165)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(220)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(275)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(330)); + + using var retry = ConfigureRetryHandler(HttpRetryConfig, mockTimeProvider, mockDelayProvider); + + using var mockResponse = new HttpResponseMessage(statusCode); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(6)); + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(50), It.IsAny()), Times.Exactly(5)); + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(6), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + } + + [Fact] + public async Task ItRetriesFewerWithMaxTotalDelayAsync() + { + // Arrange + var HttpRetryConfig = new HttpRetryConfig + { + MaxRetryCount = 5, + MinRetryDelay = TimeSpan.FromMilliseconds(50), + MaxRetryDelay = TimeSpan.FromMilliseconds(50), + MaxTotalRetryTime = TimeSpan.FromMilliseconds(100) + }; + + var mockDelayProvider = new Mock(); + var mockTimeProvider = new Mock(); + + var currentTime = DateTimeOffset.UtcNow; + mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) + .Returns(() => currentTime) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(5)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(55)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(110)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(165)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(220)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(275)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(330)); + + using var retry = ConfigureRetryHandler(HttpRetryConfig, mockTimeProvider, mockDelayProvider); + + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(4)); // 1 intial, 2 retries, 1 for logging time taken. + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(50), It.IsAny()), Times.Exactly(1)); + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + } + + [Fact] + public async Task ItRetriesFewerWithMaxTotalDelayOnExceptionAsync() + { + // Arrange + var HttpRetryConfig = new HttpRetryConfig + { + MaxRetryCount = 5, + MinRetryDelay = TimeSpan.FromMilliseconds(50), + MaxRetryDelay = TimeSpan.FromMilliseconds(50), + MaxTotalRetryTime = TimeSpan.FromMilliseconds(100) + }; + + var mockDelayProvider = new Mock(); + var mockTimeProvider = new Mock(); + + var currentTime = DateTimeOffset.UtcNow; + mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) + .Returns(() => currentTime) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(5)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(55)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(110)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(165)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(220)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(275)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(330)); + + using var retry = ConfigureRetryHandler(HttpRetryConfig, mockTimeProvider, mockDelayProvider); + var mockHandler = GetHttpMessageHandlerMock(typeof(HttpRequestException)); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + await Assert.ThrowsAsync(() => httpClient.GetAsync(new Uri("https://www.microsoft.com"), CancellationToken.None)); + + // Assert + mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(4)); // 1 intial, 2 retries, 1 for logging time taken. + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(50), It.IsAny()), Times.Exactly(1)); + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Fact] + public async Task ItRetriesOnRetryableStatusCodesAsync() + { + // Arrange + var config = new HttpRetryConfig() { RetryableStatusCodes = new List { HttpStatusCode.Unauthorized } }; + using var retry = ConfigureRetryHandler(config); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.Unauthorized); + + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); + } + + [Fact] + public async Task ItDoesNotRetryOnNonRetryableStatusCodesAsync() + { + // Arrange + var config = new HttpRetryConfig() { RetryableStatusCodes = new List { HttpStatusCode.Unauthorized } }; + using var retry = ConfigureRetryHandler(config); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); + + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + } + + [Fact] + public async Task ItRetriesOnRetryableExceptionsAsync() + { + // Arrange + var config = new HttpRetryConfig() { RetryableExceptionTypes = new List { typeof(InvalidOperationException) } }; + using var retry = ConfigureRetryHandler(config); + + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(typeof(InvalidOperationException)); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + await Assert.ThrowsAsync(async () => + await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Fact] + public async Task ItDoesNotRetryOnNonRetryableExceptionsAsync() + { + // Arrange + var config = new HttpRetryConfig() { RetryableExceptionTypes = new List { typeof(InvalidOperationException) } }; + using var retry = ConfigureRetryHandler(config); + + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(typeof(ArgumentException)); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + await Assert.ThrowsAsync(async () => + await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + } + + private static DefaultHttpRetryHandler ConfigureRetryHandler(HttpRetryConfig? config = null, + Mock? timeProvider = null, Mock? delayProvider = null) + { + delayProvider ??= new Mock(); + timeProvider ??= new Mock(); + var retry = new DefaultHttpRetryHandler(config ?? new HttpRetryConfig(), Mock.Of(), delayProvider.Object, timeProvider.Object); + return retry; + } + + private static Mock GetHttpMessageHandlerMock(HttpResponseMessage mockResponse) + { + var mockHandler = new Mock(); + mockHandler.Protected() + .Setup>("SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .ReturnsAsync(mockResponse); + return mockHandler; + } + + private static Mock GetHttpMessageHandlerMock(Type exceptionType) + { + var mockHandler = new Mock(); + mockHandler.Protected() + .Setup>("SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .ThrowsAsync(Activator.CreateInstance(exceptionType) as Exception); + return mockHandler; + } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/Reliability/HttpRetryConfigTests.cs b/dotnet/src/SemanticKernel.UnitTests/Reliability/HttpRetryConfigTests.cs new file mode 100644 index 000000000000..e57c238df8d3 --- /dev/null +++ b/dotnet/src/SemanticKernel.UnitTests/Reliability/HttpRetryConfigTests.cs @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Configuration; +using Microsoft.SemanticKernel.Reliability; +using Xunit; + +namespace SemanticKernel.UnitTests.Reliability; + +/// +/// Unit tests of . +/// +public class HttpRetryConfigTests +{ + [Fact] + public async Task NegativeMaxRetryCountThrowsAsync() + { + // Act + await Assert.ThrowsAsync(() => + { + var httpRetryConfig = new HttpRetryConfig() { MaxRetryCount = -1 }; + return Task.CompletedTask; + }); + } + + [Fact] + public void SetDefaultHttpRetryConfig() + { + // Arrange + var config = new KernelConfig(); + var httpRetryConfig = new HttpRetryConfig() { MaxRetryCount = 1 }; + + // Act + config.SetDefaultHttpRetryConfig(httpRetryConfig); + + // Assert + Assert.Equal(httpRetryConfig, config.DefaultHttpRetryConfig); + } + + [Fact] + public void SetDefaultHttpRetryConfigToDefaultIfNotSet() + { + // Arrange + var config = new KernelConfig(); + + // Act + // Assert + var defaultConfig = new HttpRetryConfig(); + Assert.Equal(defaultConfig.MaxRetryCount, config.DefaultHttpRetryConfig.MaxRetryCount); + Assert.Equal(defaultConfig.MaxRetryDelay, config.DefaultHttpRetryConfig.MaxRetryDelay); + Assert.Equal(defaultConfig.MinRetryDelay, config.DefaultHttpRetryConfig.MinRetryDelay); + Assert.Equal(defaultConfig.MaxTotalRetryTime, config.DefaultHttpRetryConfig.MaxTotalRetryTime); + Assert.Equal(defaultConfig.UseExponentialBackoff, config.DefaultHttpRetryConfig.UseExponentialBackoff); + Assert.Equal(defaultConfig.RetryableStatusCodes, config.DefaultHttpRetryConfig.RetryableStatusCodes); + Assert.Equal(defaultConfig.RetryableExceptionTypes, config.DefaultHttpRetryConfig.RetryableExceptionTypes); + } + + [Fact] + public void SetDefaultHttpRetryConfigToDefaultIfNull() + { + // Arrange + var config = new KernelConfig(); + + // Act + config.SetDefaultHttpRetryConfig(null); + + // Assert + var defaultConfig = new HttpRetryConfig(); + Assert.Equal(defaultConfig.MaxRetryCount, config.DefaultHttpRetryConfig.MaxRetryCount); + Assert.Equal(defaultConfig.MaxRetryDelay, config.DefaultHttpRetryConfig.MaxRetryDelay); + Assert.Equal(defaultConfig.MinRetryDelay, config.DefaultHttpRetryConfig.MinRetryDelay); + Assert.Equal(defaultConfig.MaxTotalRetryTime, config.DefaultHttpRetryConfig.MaxTotalRetryTime); + Assert.Equal(defaultConfig.UseExponentialBackoff, config.DefaultHttpRetryConfig.UseExponentialBackoff); + Assert.Equal(defaultConfig.RetryableStatusCodes, config.DefaultHttpRetryConfig.RetryableStatusCodes); + Assert.Equal(defaultConfig.RetryableExceptionTypes, config.DefaultHttpRetryConfig.RetryableExceptionTypes); + } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/Reliability/NullHttpRetryHandlerTests.cs b/dotnet/src/SemanticKernel.UnitTests/Reliability/NullHttpRetryHandlerTests.cs new file mode 100644 index 000000000000..c6f6fee99501 --- /dev/null +++ b/dotnet/src/SemanticKernel.UnitTests/Reliability/NullHttpRetryHandlerTests.cs @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Reliability; +using Moq; +using Moq.Protected; +using Xunit; + +namespace SemanticKernel.UnitTests.Reliability; + +public class NullHttpRetryHandlerTests +{ + [Fact] + public async Task ItDoesNotRetryOnExceptionAsync() + { + // Arrange + using var retry = new NullHttpRetryHandler(); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + } + + [Fact] + public async Task NoExceptionNoRetryAsync() + { + // Arrange + using var retry = new NullHttpRetryHandler(); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.OK); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + } + + [Fact] + public async Task TaskCanceledExceptionThrownOnCancellationTokenAsync() + { + // Arrange + using var retry = new NullHttpRetryHandler(); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + using var cancellationTokenSource = new CancellationTokenSource(); + cancellationTokenSource.Cancel(); + + // Act + await Assert.ThrowsAsync(async () => + await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, cancellationTokenSource.Token)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Fact] + public async Task ItDoestExecuteOnFalseCancellationTokenAsync() + { + // Arrange + using var retry = new NullHttpRetryHandler(); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, new CancellationToken(false)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + } + + private static Mock GetHttpMessageHandlerMock(HttpResponseMessage mockResponse) + { + var mockHandler = new Mock(); + mockHandler.Protected() + .Setup>("SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .ReturnsAsync(mockResponse); + return mockHandler; + } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/Reliability/PassThroughWithoutRetryTests.cs b/dotnet/src/SemanticKernel.UnitTests/Reliability/PassThroughWithoutRetryTests.cs deleted file mode 100644 index 098fe632f754..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/Reliability/PassThroughWithoutRetryTests.cs +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.AI; -using Microsoft.SemanticKernel.Reliability; -using Moq; -using Xunit; - -namespace SemanticKernel.UnitTests.Reliability; - -public class PassThroughWithoutRetryTests -{ - [Fact] - public async Task ItDoesNotRetryOnExceptionAsync() - { - // Arrange - var retry = new PassThroughWithoutRetry(); - var action = new Mock>(); - action.Setup(a => a()).Throws(new AIException(AIException.ErrorCodes.Throttling, "Throttling Test")); - - // Act - await Assert.ThrowsAsync(() => retry.ExecuteWithRetryAsync(action.Object, Mock.Of())); - - // Assert - action.Verify(a => a(), Times.Once); - } - - [Fact] - public async Task NoExceptionNoRetryAsync() - { - // Arrange - var log = new Mock(); - var retry = new PassThroughWithoutRetry(); - var action = new Mock>(); - - // Act - await retry.ExecuteWithRetryAsync(action.Object, log.Object); - - // Assert - action.Verify(a => a(), Times.Once); - } - - [Fact] - public async Task ItDoesNotExecuteOnCancellationTokenAsync() - { - // Arrange - var retry = new PassThroughWithoutRetry(); - var action = new Mock>(); - action.Setup(a => a()).Throws(new AIException(AIException.ErrorCodes.Throttling, "Throttling Test")); - - // Act - await retry.ExecuteWithRetryAsync(action.Object, Mock.Of(), new CancellationToken(true)); - - // Assert - action.Verify(a => a(), Times.Never); - } - - [Fact] - public async Task ItDoestExecuteOnFalseCancellationTokenAsync() - { - // Arrange - var retry = new PassThroughWithoutRetry(); - var action = new Mock>(); - action.Setup(a => a()).Throws(new AIException(AIException.ErrorCodes.Throttling, "Throttling Test")); - - // Act - await Assert.ThrowsAsync(() => retry.ExecuteWithRetryAsync(action.Object, Mock.Of(), new CancellationToken(false))); - - // Assert - action.Verify(a => a(), Times.Once); - } -} diff --git a/dotnet/src/SemanticKernel/AI/ITextCompletionClient.cs b/dotnet/src/SemanticKernel/AI/ITextCompletionClient.cs index a4118375d8db..833c6b495c6f 100644 --- a/dotnet/src/SemanticKernel/AI/ITextCompletionClient.cs +++ b/dotnet/src/SemanticKernel/AI/ITextCompletionClient.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Threading; using System.Threading.Tasks; namespace Microsoft.SemanticKernel.AI; @@ -14,6 +15,7 @@ public interface ITextCompletionClient /// /// The prompt to complete. /// Request settings for the completion API + /// Cancellation token /// Text generated by the remote model - public Task CompleteAsync(string text, CompleteRequestSettings requestSettings); + public Task CompleteAsync(string text, CompleteRequestSettings requestSettings, CancellationToken cancellationToken = default); } diff --git a/dotnet/src/SemanticKernel/AI/OpenAI/Clients/AzureOpenAIClientAbstract.cs b/dotnet/src/SemanticKernel/AI/OpenAI/Clients/AzureOpenAIClientAbstract.cs index f3ab80244db7..3ac693ed051f 100644 --- a/dotnet/src/SemanticKernel/AI/OpenAI/Clients/AzureOpenAIClientAbstract.cs +++ b/dotnet/src/SemanticKernel/AI/OpenAI/Clients/AzureOpenAIClientAbstract.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel.AI.OpenAI.HttpSchema; +using Microsoft.SemanticKernel.Reliability; using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.AI.OpenAI.Clients; @@ -25,7 +26,7 @@ public abstract class AzureOpenAIClientAbstract : OpenAIClientAbstract /// protected string AzureOpenAIApiVersion { - get { return this._azureOpenAIApiVersion; } + get => this._azureOpenAIApiVersion; set { if (string.IsNullOrWhiteSpace(value)) @@ -48,7 +49,8 @@ protected string AzureOpenAIApiVersion /// Construct an AzureOpenAIClientAbstract object /// /// Logger - protected AzureOpenAIClientAbstract(ILogger? log = null) : base(log) + /// Retry handler factory + protected AzureOpenAIClientAbstract(ILogger? log = null, IDelegatingHandlerFactory? handlerFactory = null) : base(log, handlerFactory) { } @@ -161,5 +163,5 @@ protected async Task CacheDeploymentsAsync() private string _azureOpenAIApiVersion = DefaultAzureAPIVersion; - #endregion + #endregion private ================================================================================ } diff --git a/dotnet/src/SemanticKernel/AI/OpenAI/Clients/OpenAIClientAbstract.cs b/dotnet/src/SemanticKernel/AI/OpenAI/Clients/OpenAIClientAbstract.cs index 869cb9de2e41..5e9df2d923ea 100644 --- a/dotnet/src/SemanticKernel/AI/OpenAI/Clients/OpenAIClientAbstract.cs +++ b/dotnet/src/SemanticKernel/AI/OpenAI/Clients/OpenAIClientAbstract.cs @@ -7,11 +7,13 @@ using System.Net.Http; using System.Net.Mime; using System.Text; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.SemanticKernel.AI.Embeddings; using Microsoft.SemanticKernel.AI.OpenAI.HttpSchema; +using Microsoft.SemanticKernel.Reliability; using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.AI.OpenAI.Clients; @@ -33,14 +35,19 @@ public abstract class OpenAIClientAbstract : IDisposable protected HttpClient HTTPClient { get; } private readonly HttpClientHandler _httpClientHandler; + private readonly IDelegatingHandlerFactory _handlerFactory; + private readonly DelegatingHandler _retryHandler; - internal OpenAIClientAbstract(ILogger? log = null) + internal OpenAIClientAbstract(ILogger? log = null, IDelegatingHandlerFactory? handlerFactory = null) { - if (log != null) { this.Log = log; } + this.Log = log ?? this.Log; + this._handlerFactory = handlerFactory ?? new DefaultHttpRetryHandlerFactory(); - // TODO: allow injection of retry logic, e.g. Polly this._httpClientHandler = new() { CheckCertificateRevocationList = true }; - this.HTTPClient = new HttpClient(this._httpClientHandler); + this._retryHandler = this._handlerFactory.Create(this.Log); + this._retryHandler.InnerHandler = this._httpClientHandler; + + this.HTTPClient = new HttpClient(this._retryHandler); this.HTTPClient.DefaultRequestHeaders.Add("User-Agent", HTTPUseragent); } @@ -49,15 +56,16 @@ internal OpenAIClientAbstract(ILogger? log = null) /// /// URL for the completion request API /// Prompt to complete + /// Cancellation token /// The completed text /// AIException thrown during the request. - protected async Task ExecuteCompleteRequestAsync(string url, string requestBody) + protected async Task ExecuteCompleteRequestAsync(string url, string requestBody, CancellationToken cancellationToken = default) { try { this.Log.LogDebug("Sending completion request to {0}: {1}", url, requestBody); - var result = await this.ExecutePostRequestAsync(url, requestBody); + var result = await this.ExecutePostRequestAsync(url, requestBody, cancellationToken); if (result.Completions.Count < 1) { throw new AIException( @@ -124,6 +132,7 @@ protected virtual void Dispose(bool disposing) { this.HTTPClient.Dispose(); this._httpClientHandler.Dispose(); + this._retryHandler.Dispose(); } } @@ -132,14 +141,14 @@ protected virtual void Dispose(bool disposing) // HTTP user agent sent to remote endpoints private const string HTTPUseragent = "Microsoft Semantic Kernel"; - private async Task ExecutePostRequestAsync(string url, string requestBody) + private async Task ExecutePostRequestAsync(string url, string requestBody, CancellationToken cancellationToken = default) { string responseJson; try { using HttpContent content = new StringContent(requestBody, Encoding.UTF8, MediaTypeNames.Application.Json); - HttpResponseMessage response = await this.HTTPClient.PostAsync(url, content); + HttpResponseMessage response = await this.HTTPClient.PostAsync(url, content, cancellationToken); if (response == null) { diff --git a/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextCompletion.cs b/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextCompletion.cs index 38e8055a47c4..0eb7c96d0896 100644 --- a/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextCompletion.cs +++ b/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextCompletion.cs @@ -1,10 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel.AI.OpenAI.Clients; using Microsoft.SemanticKernel.AI.OpenAI.HttpSchema; using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Reliability; using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.AI.OpenAI.Services; @@ -22,8 +24,15 @@ public sealed class AzureTextCompletion : AzureOpenAIClientAbstract, ITextComple /// Azure OpenAI API key, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// Azure OpenAI API version, see https://learn.microsoft.com/azure/cognitive-services/openai/reference /// Application logger - public AzureTextCompletion(string modelId, string endpoint, string apiKey, string apiVersion, ILogger? log = null) - : base(log) + /// Retry handler factory for HTTP requests. + public AzureTextCompletion( + string modelId, + string endpoint, + string apiKey, + string apiVersion, + ILogger? log = null, + IDelegatingHandlerFactory? handlerFactory = null) + : base(log, handlerFactory) { Verify.NotEmpty(modelId, "The ID cannot be empty, you must provide a Model ID or a Deployment name."); this._modelId = modelId; @@ -43,9 +52,10 @@ public AzureTextCompletion(string modelId, string endpoint, string apiKey, strin /// /// Text to complete /// Request settings for the completion API + /// Cancellation token /// The completed text. /// AIException thrown during the request - public async Task CompleteAsync(string text, CompleteRequestSettings requestSettings) + public async Task CompleteAsync(string text, CompleteRequestSettings requestSettings, CancellationToken cancellationToken = default) { Verify.NotNull(requestSettings, "Completion settings cannot be empty"); @@ -72,7 +82,7 @@ public async Task CompleteAsync(string text, CompleteRequestSettings req Stop = requestSettings.StopSequences is { Count: > 0 } ? requestSettings.StopSequences : null, }); - return await this.ExecuteCompleteRequestAsync(url, requestBody); + return await this.ExecuteCompleteRequestAsync(url, requestBody, cancellationToken); } #region private ================================================================================ diff --git a/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextEmbeddings.cs b/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextEmbeddings.cs index e7c2dfac1a51..283d56d5b3db 100644 --- a/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextEmbeddings.cs +++ b/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextEmbeddings.cs @@ -7,6 +7,7 @@ using Microsoft.SemanticKernel.AI.OpenAI.Clients; using Microsoft.SemanticKernel.AI.OpenAI.HttpSchema; using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Reliability; using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.AI.OpenAI.Services; @@ -26,8 +27,10 @@ public sealed class AzureTextEmbeddings : AzureOpenAIClientAbstract, IEmbeddingG /// Azure OpenAI API key, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// Azure OpenAI API version, see https://learn.microsoft.com/azure/cognitive-services/openai/reference /// Application logger - public AzureTextEmbeddings(string modelId, string endpoint, string apiKey, string apiVersion, ILogger? log = null) - : base(log) + /// An optional HTTP retry handler factory + public AzureTextEmbeddings(string modelId, string endpoint, string apiKey, string apiVersion, ILogger? log = null, + IDelegatingHandlerFactory? handlerFactory = null) + : base(log, handlerFactory) { Verify.NotEmpty(modelId, "The ID cannot be empty, you must provide a Model ID or a Deployment name."); this._modelId = modelId; diff --git a/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextCompletion.cs b/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextCompletion.cs index 00f6553f7ef2..95ec72da38da 100644 --- a/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextCompletion.cs +++ b/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextCompletion.cs @@ -1,11 +1,13 @@ // Copyright (c) Microsoft. All rights reserved. using System.Net.Http.Headers; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel.AI.OpenAI.Clients; using Microsoft.SemanticKernel.AI.OpenAI.HttpSchema; using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Reliability; using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.AI.OpenAI.Services; @@ -27,8 +29,10 @@ public sealed class OpenAITextCompletion : OpenAIClientAbstract, ITextCompletion /// OpenAI API key, see https://platform.openai.com/account/api-keys /// OpenAI organization id. This is usually optional unless your account belongs to multiple organizations. /// Logger - public OpenAITextCompletion(string modelId, string apiKey, string? organization = null, ILogger? log = null) : - base(log) + /// Retry handler + public OpenAITextCompletion(string modelId, string apiKey, string? organization = null, ILogger? log = null, + IDelegatingHandlerFactory? handlerFactory = null) : + base(log, handlerFactory) { Verify.NotEmpty(modelId, "The OpenAI model ID cannot be empty"); this._modelId = modelId; @@ -47,9 +51,10 @@ public OpenAITextCompletion(string modelId, string apiKey, string? organization /// /// The prompt to complete. /// Request settings for the completion API + /// Cancellation token /// The completed text /// AIException thrown during the request - public async Task CompleteAsync(string text, CompleteRequestSettings requestSettings) + public async Task CompleteAsync(string text, CompleteRequestSettings requestSettings, CancellationToken cancellationToken = default) { Verify.NotNull(requestSettings, "Completion settings cannot be empty"); @@ -74,6 +79,6 @@ public async Task CompleteAsync(string text, CompleteRequestSettings req Stop = requestSettings.StopSequences is { Count: > 0 } ? requestSettings.StopSequences : null, }); - return await this.ExecuteCompleteRequestAsync(url, requestBody); + return await this.ExecuteCompleteRequestAsync(url, requestBody, cancellationToken); } } diff --git a/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextEmbeddings.cs b/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextEmbeddings.cs index 6506f4710620..039f0d937c57 100644 --- a/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextEmbeddings.cs +++ b/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextEmbeddings.cs @@ -8,6 +8,7 @@ using Microsoft.SemanticKernel.AI.OpenAI.Clients; using Microsoft.SemanticKernel.AI.OpenAI.HttpSchema; using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Reliability; using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.AI.OpenAI.Services; @@ -30,8 +31,10 @@ public sealed class OpenAITextEmbeddings : OpenAIClientAbstract, IEmbeddingGener /// OpenAI API Key /// Optional OpenAI organization ID, usually required only if your account belongs to multiple organizations /// Application logger - public OpenAITextEmbeddings(string modelId, string apiKey, string? organization = null, ILogger? log = null) - : base(log) + /// Retry handler factory for HTTP requests. + public OpenAITextEmbeddings(string modelId, string apiKey, string? organization = null, ILogger? log = null, + IDelegatingHandlerFactory? handlerFactory = null) + : base(log, handlerFactory) { Verify.NotEmpty(modelId, "The OpenAI model ID cannot be empty"); this._modelId = modelId; diff --git a/dotnet/src/SemanticKernel/Configuration/KernelConfig.cs b/dotnet/src/SemanticKernel/Configuration/KernelConfig.cs index ef8cf8cff877..1c73bf0821f2 100644 --- a/dotnet/src/SemanticKernel/Configuration/KernelConfig.cs +++ b/dotnet/src/SemanticKernel/Configuration/KernelConfig.cs @@ -15,9 +15,14 @@ namespace Microsoft.SemanticKernel.Configuration; public sealed class KernelConfig { /// - /// Global retry logic used for all the backends + /// Factory for creating HTTP handlers. /// - public IRetryMechanism RetryMechanism { get => this._retryMechanism; } + public IDelegatingHandlerFactory HttpHandlerFactory { get; private set; } = new DefaultHttpRetryHandlerFactory(new HttpRetryConfig()); + + /// + /// Default HTTP retry configuration for built-in HTTP handler factory. + /// + public HttpRetryConfig DefaultHttpRetryConfig { get; private set; } = new(); /// /// Adds an Azure OpenAI backend to the list. @@ -180,13 +185,28 @@ public bool HasEmbeddingsBackend(string label, Func? condi } /// - /// Set the retry mechanism to use for the kernel. + /// Set the http retry handler factory to use for the kernel. /// - /// Retry mechanism to use. + /// Http retry handler factory to use. /// The updated kernel configuration. - public KernelConfig SetRetryMechanism(IRetryMechanism? retryMechanism = null) + public KernelConfig SetHttpRetryHandlerFactory(IDelegatingHandlerFactory? httpHandlerFactory = null) { - this._retryMechanism = retryMechanism ?? new PassThroughWithoutRetry(); + if (httpHandlerFactory != null) + { + this.HttpHandlerFactory = httpHandlerFactory; + } + + return this; + } + + public KernelConfig SetDefaultHttpRetryConfig(HttpRetryConfig? httpRetryConfig) + { + if (httpRetryConfig != null) + { + this.DefaultHttpRetryConfig = httpRetryConfig; + this.SetHttpRetryHandlerFactory(new DefaultHttpRetryHandlerFactory(httpRetryConfig)); + } + return this; } @@ -252,7 +272,7 @@ public IBackendConfig GetCompletionBackend(string? label = null) { throw new KernelException( KernelException.ErrorCodes.BackendNotFound, - $"A label was not provided and no default completion backend is available."); + "A label was not provided and no default completion backend is available."); } return this.CompletionBackends[this._defaultCompletionBackend]; @@ -287,7 +307,7 @@ public IBackendConfig GetEmbeddingsBackend(string? label = null) { throw new KernelException( KernelException.ErrorCodes.BackendNotFound, - $"A label was not provided and no default embeddings backend is available."); + "A label was not provided and no default embeddings backend is available."); } return this.EmbeddingsBackends[this._defaultEmbeddingsBackend]; @@ -393,11 +413,10 @@ public KernelConfig RemoveAllBackends() #region private - private Dictionary CompletionBackends { get; set; } = new(); - private Dictionary EmbeddingsBackends { get; set; } = new(); + private Dictionary CompletionBackends { get; } = new(); + private Dictionary EmbeddingsBackends { get; } = new(); private string? _defaultCompletionBackend; private string? _defaultEmbeddingsBackend; - private IRetryMechanism _retryMechanism = new PassThroughWithoutRetry(); #endregion } diff --git a/dotnet/src/SemanticKernel/Kernel.cs b/dotnet/src/SemanticKernel/Kernel.cs index a9903586cb5f..4cec17592ea7 100644 --- a/dotnet/src/SemanticKernel/Kernel.cs +++ b/dotnet/src/SemanticKernel/Kernel.cs @@ -172,10 +172,7 @@ public async Task RunAsync(ContextVariables variables, CancellationTo try { cancellationToken.ThrowIfCancellationRequested(); - await this._config.RetryMechanism.ExecuteWithRetryAsync( - async () => { context = await f.InvokeAsync(context); }, - this._log, - cancellationToken); + context = await f.InvokeAsync(context); if (context.ErrorOccurred) { @@ -266,7 +263,8 @@ private ISKFunction CreateSemanticFunction( azureBackendConfig.Endpoint, azureBackendConfig.APIKey, azureBackendConfig.APIVersion, - this._log)); + this._log, + this._config.HttpHandlerFactory)); break; case OpenAIConfig openAiConfig: @@ -274,7 +272,8 @@ private ISKFunction CreateSemanticFunction( openAiConfig.ModelId, openAiConfig.APIKey, openAiConfig.OrgId, - this._log)); + this._log, + this._config.HttpHandlerFactory)); break; default: diff --git a/dotnet/src/SemanticKernel/KernelBuilder.cs b/dotnet/src/SemanticKernel/KernelBuilder.cs index d039eb10e671..27aca197704e 100644 --- a/dotnet/src/SemanticKernel/KernelBuilder.cs +++ b/dotnet/src/SemanticKernel/KernelBuilder.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.Net.Http; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.SemanticKernel.AI.Embeddings; @@ -8,6 +9,7 @@ using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.KernelExtensions; using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Reliability; using Microsoft.SemanticKernel.SkillDefinition; using Microsoft.SemanticKernel.TemplateEngine; @@ -23,6 +25,7 @@ public sealed class KernelBuilder private ISemanticTextMemory _memory = NullMemory.Instance; private ILogger _log = NullLogger.Instance; private IMemoryStore? _memoryStorage = null; + private IDelegatingHandlerFactory? _httpHandlerFactory = null; /// /// Create a new kernel instance @@ -40,6 +43,11 @@ public static IKernel Create() /// Kernel instance public IKernel Build() { + if (this._httpHandlerFactory != null) + { + this._config.SetHttpRetryHandlerFactory(this._httpHandlerFactory); + } + var instance = new Kernel( new SkillCollection(this._log), new PromptTemplateEngine(this._log), @@ -108,6 +116,18 @@ public KernelBuilder WithMemoryStorageAndEmbeddingGenerator( return this; } + /// + /// Add a retry handler factory to the kernel to be built. + /// + /// Retry handler factory to add. + /// Updated kernel builder including the retry handler factory. + public KernelBuilder WithRetryHandlerFactory(IDelegatingHandlerFactory httpHandlerFactory) + { + Verify.NotNull(httpHandlerFactory, "The retry handler factory instance provided is NULL"); + this._httpHandlerFactory = httpHandlerFactory; + return this; + } + /// /// Use the given configuration with the kernel to be built. /// diff --git a/dotnet/src/SemanticKernel/KernelExtensions/MemoryConfiguration.cs b/dotnet/src/SemanticKernel/KernelExtensions/MemoryConfiguration.cs index eb54f57f5056..76430561e22c 100644 --- a/dotnet/src/SemanticKernel/KernelExtensions/MemoryConfiguration.cs +++ b/dotnet/src/SemanticKernel/KernelExtensions/MemoryConfiguration.cs @@ -51,7 +51,8 @@ public static void UseMemory(this IKernel kernel, string? embeddingsBackendLabel azureAIConfig.Endpoint, azureAIConfig.APIKey, azureAIConfig.APIVersion, - kernel.Log); + kernel.Log, + kernel.Config.HttpHandlerFactory); break; case OpenAIConfig openAIConfig: @@ -59,7 +60,8 @@ public static void UseMemory(this IKernel kernel, string? embeddingsBackendLabel openAIConfig.ModelId, openAIConfig.APIKey, openAIConfig.OrgId, - kernel.Log); + kernel.Log, + kernel.Config.HttpHandlerFactory); break; default: diff --git a/dotnet/src/SemanticKernel/Orchestration/SKFunction.cs b/dotnet/src/SemanticKernel/Orchestration/SKFunction.cs index e18a5496dbc7..466bbf9e4996 100644 --- a/dotnet/src/SemanticKernel/Orchestration/SKFunction.cs +++ b/dotnet/src/SemanticKernel/Orchestration/SKFunction.cs @@ -109,7 +109,7 @@ async Task LocalFunc( { string prompt = await functionConfig.PromptTemplate.RenderAsync(context); - string completion = await client.CompleteAsync(prompt, requestSettings); + string completion = await client.CompleteAsync(prompt, requestSettings, context.CancellationToken); context.Variables.Update(completion); } catch (Exception ex) when (!ex.IsCriticalException()) diff --git a/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandler.cs b/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandler.cs new file mode 100644 index 000000000000..c921d6870b6c --- /dev/null +++ b/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandler.cs @@ -0,0 +1,232 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Microsoft.SemanticKernel.Reliability; + +public sealed class DefaultHttpRetryHandler : DelegatingHandler +{ + /// + /// Initializes a new instance of the class. + /// + /// The retry configuration. + /// The logger. + public DefaultHttpRetryHandler(HttpRetryConfig? config = null, ILogger? log = null) + : this(config ?? new HttpRetryConfig(), log, null, null) + { + } + + internal DefaultHttpRetryHandler( + HttpRetryConfig config, + ILogger? log = null, + IDelayProvider? delayProvider = null, + ITimeProvider? timeProvider = null) + { + this._config = config; + this._log = log ?? NullLogger.Instance; + this._delayProvider = delayProvider ?? new TaskDelayProvider(); + this._timeProvider = timeProvider ?? new DefaultTimeProvider(); + } + + /// + /// Executes the action with retry logic + /// + /// + /// The request is retried if it throws an exception that is a retryable exception. + /// If the request throws an exception that is not a retryable exception, it is not retried. + /// If the request returns a response with a retryable error code, it is retried. + /// If the request returns a response with a non-retryable error code, it is not retried. + /// If the exception contains a RetryAfter header, the request is retried after the specified delay. + /// If configured to use exponential backoff, the delay is doubled for each retry. + /// + /// The request. + /// The cancellation token. + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + int retryCount = 0; + + var start = this._timeProvider.GetCurrentTime(); + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + TimeSpan waitFor; + string reason; + HttpResponseMessage? response = null; + try + { + response = await base.SendAsync(request, cancellationToken); + + // If the request does not require a retry then we're done + if (!this.ShouldRetry(response.StatusCode)) + { + return response; + } + + reason = response.StatusCode.ToString(); + + // If the retry count is greater than the max retry count then we'll + // just return + if (retryCount >= this._config.MaxRetryCount) + { + this._log.LogError( + "Error executing request, max retry count reached. Reason: {0}", reason); + return response; + } + + // If the retry delay is longer than the total timeout, then we'll + // just return + if (!this.HasTimeForRetry(start, retryCount, response, out waitFor)) + { + var timeTaken = this._timeProvider.GetCurrentTime() - start; + this._log.LogError( + "Error executing request, max total retry time reached. Reason: {0}. Time spent: {1}ms", reason, + timeTaken.TotalMilliseconds); + return response; + } + } + catch (Exception e) when (this.ShouldRetry(e) || this.ShouldRetry(e.InnerException)) + { + reason = e.GetType().ToString(); + if (retryCount >= this._config.MaxRetryCount) + { + this._log.LogError(e, + "Error executing request, max retry count reached. Reason: {0}", reason); + throw; + } + else if (!this.HasTimeForRetry(start, retryCount, response, out waitFor)) + { + var timeTaken = this._timeProvider.GetCurrentTime() - start; + this._log.LogError( + "Error executing request, max total retry time reached. Reason: {0}. Time spent: {1}ms", reason, + timeTaken.TotalMilliseconds); + throw; + } + } + + // If the request requires a retry then we'll retry + this._log.LogWarning( + "Error executing action [attempt {0} of {1}]. Reason: {2}. Will retry after {3}ms", + retryCount + 1, + this._config.MaxRetryCount, + reason, + waitFor.TotalMilliseconds); + + // Increase retryCount + retryCount++; + + response?.Dispose(); + + // Delay + await this._delayProvider.DelayAsync(waitFor, cancellationToken).ConfigureAwait(false); + } + } + + /// + /// Get the wait time for the next retry. + /// + /// Current retry count + /// The response message that potentially contains RetryAfter header. + private TimeSpan GetWaitTime(int retryCount, HttpResponseMessage? response) + { + // If the response contains a RetryAfter header, use that value + // Otherwise, use the configured min retry delay + var retryAfter = response?.Headers.RetryAfter?.Date.HasValue == true + ? response?.Headers.RetryAfter?.Date - this._timeProvider.GetCurrentTime() + : (response?.Headers.RetryAfter?.Delta) ?? this._config.MinRetryDelay; + retryAfter ??= this._config.MinRetryDelay; + + // If the retry delay is longer than the max retry delay, use the max retry delay + var timeToWait = retryAfter > this._config.MaxRetryDelay + ? this._config.MaxRetryDelay + : retryAfter < this._config.MinRetryDelay + ? this._config.MinRetryDelay + : retryAfter ?? default; + + // If exponential backoff is enabled, double the delay for each retry + if (this._config.UseExponentialBackoff) + { + for (var backoffRetryCount = 1; backoffRetryCount < retryCount + 1; backoffRetryCount++) + { + timeToWait = timeToWait.Add(timeToWait); + } + } + + return timeToWait; + } + + /// + /// Determines if there is time left for a retry. + /// + /// The start time of the original request. + /// The current retry count. + /// The response message that potentially contains RetryAfter header. + /// The wait time for the next retry. + /// True if there is time left for a retry, false otherwise. + private bool HasTimeForRetry(DateTimeOffset start, int retryCount, HttpResponseMessage? response, out TimeSpan waitFor) + { + waitFor = this.GetWaitTime(retryCount, response); + var currentTIme = this._timeProvider.GetCurrentTime(); + var result = currentTIme - start + waitFor; + + return result < this._config.MaxTotalRetryTime; + } + + private bool ShouldRetry(HttpStatusCode statusCode) + { + return this._config.RetryableStatusCodes.Contains(statusCode); + } + + private bool ShouldRetry(Exception? exception) + { + if (exception == null) + { + return false; + } + + return this._config.RetryableExceptionTypes.Contains(exception.GetType()); + } + + private readonly HttpRetryConfig _config; + private readonly ILogger _log; + private readonly IDelayProvider _delayProvider; + private readonly ITimeProvider _timeProvider; + + /// + /// Interface for a delay provider, primarily to enable unit testing. + /// + internal interface IDelayProvider + { + Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken); + } + + internal class TaskDelayProvider : IDelayProvider + { + public async Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken) + { + await Task.Delay(delay, cancellationToken); + } + } + + /// + /// Interface for a time provider, primarily to enable unit testing. + /// + internal interface ITimeProvider + { + DateTimeOffset GetCurrentTime(); + } + + internal class DefaultTimeProvider : ITimeProvider + { + public DateTimeOffset GetCurrentTime() + { + return DateTimeOffset.UtcNow; + } + } +} diff --git a/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandlerFactory.cs b/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandlerFactory.cs new file mode 100644 index 000000000000..3c457fda68b7 --- /dev/null +++ b/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandlerFactory.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Microsoft.Extensions.Logging; + +namespace Microsoft.SemanticKernel.Reliability; + +public class DefaultHttpRetryHandlerFactory : IDelegatingHandlerFactory +{ + public DefaultHttpRetryHandlerFactory(HttpRetryConfig? config = null) + { + this._config = config; + } + + public DelegatingHandler Create(ILogger log) + { + return new DefaultHttpRetryHandler(this._config, log); + } + + private readonly HttpRetryConfig? _config; +} diff --git a/dotnet/src/SemanticKernel/Reliability/HttpRetryConfig.cs b/dotnet/src/SemanticKernel/Reliability/HttpRetryConfig.cs new file mode 100644 index 000000000000..3b1a8b53fb21 --- /dev/null +++ b/dotnet/src/SemanticKernel/Reliability/HttpRetryConfig.cs @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; + +namespace Microsoft.SemanticKernel.Reliability; + +/// +/// Retry configuration for IHttpRetryPolicy that uses RetryAfter header when present. +/// +public sealed class HttpRetryConfig +{ + /// + /// Maximum number of retries. + /// + /// Thrown when value is negative. + public int MaxRetryCount + { + get => this._maxRetryCount; + set + { + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(this.MaxRetryCount), "Max retry count cannot be negative."); + } + + this._maxRetryCount = value; + } + } + + /// + /// Minimum delay between retries. + /// + public TimeSpan MinRetryDelay { get; set; } = TimeSpan.FromSeconds(2); + + /// + /// Maximum delay between retries. + /// + public TimeSpan MaxRetryDelay { get; set; } = TimeSpan.FromSeconds(60); + + /// + /// Maximum total time spent retrying. + /// + public TimeSpan MaxTotalRetryTime { get; set; } = TimeSpan.FromMinutes(2); + + /// + /// Whether to use exponential backoff or not. + /// + public bool UseExponentialBackoff { get; set; } + + /// + /// List of status codes that should be retried. + /// + public List RetryableStatusCodes { get; set; } = new() + { + HttpStatusCode.RequestTimeout, + HttpStatusCode.ServiceUnavailable, + HttpStatusCode.GatewayTimeout, + HttpStatusCode.TooManyRequests + }; + + /// + /// List of exception types that should be retried. + /// + public List RetryableExceptionTypes { get; set; } = new() + { + typeof(HttpRequestException) + }; + + private int _maxRetryCount = 1; +} diff --git a/dotnet/src/SemanticKernel/Reliability/IDelegatingHandlerFactory.cs b/dotnet/src/SemanticKernel/Reliability/IDelegatingHandlerFactory.cs new file mode 100644 index 000000000000..1d4d4d29f8ae --- /dev/null +++ b/dotnet/src/SemanticKernel/Reliability/IDelegatingHandlerFactory.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Microsoft.Extensions.Logging; + +namespace Microsoft.SemanticKernel.Reliability; + +/// +/// Factory for creating instances. +/// +public interface IDelegatingHandlerFactory +{ + DelegatingHandler Create(ILogger log); +} diff --git a/dotnet/src/SemanticKernel/Reliability/IRetryMechanism.cs b/dotnet/src/SemanticKernel/Reliability/IRetryMechanism.cs deleted file mode 100644 index cd92fc8e278c..000000000000 --- a/dotnet/src/SemanticKernel/Reliability/IRetryMechanism.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; - -namespace Microsoft.SemanticKernel.Reliability; - -/// -/// Interface for retry mechanisms on AI calls. -/// -public interface IRetryMechanism -{ - /// - /// Executes the given action with retry logic. - /// - /// The action to retry on exception. - /// The logger to use. - /// The cancellation token. - /// An awaitable task. - Task ExecuteWithRetryAsync(Func action, ILogger log, CancellationToken cancellationToken = default); -} diff --git a/dotnet/src/SemanticKernel/Reliability/NullHttpRetryHandler.cs b/dotnet/src/SemanticKernel/Reliability/NullHttpRetryHandler.cs new file mode 100644 index 000000000000..8749ace9ef51 --- /dev/null +++ b/dotnet/src/SemanticKernel/Reliability/NullHttpRetryHandler.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Microsoft.Extensions.Logging; + +namespace Microsoft.SemanticKernel.Reliability; + +public class NullHttpRetryHandlerFactory : IDelegatingHandlerFactory +{ + public DelegatingHandler Create(ILogger log) + { + return new NullHttpRetryHandler(); + } +} + +/// +/// A http retry handler that does not retry. +/// +public class NullHttpRetryHandler : DelegatingHandler +{ +} diff --git a/dotnet/src/SemanticKernel/Reliability/PassThroughWithoutRetry.cs b/dotnet/src/SemanticKernel/Reliability/PassThroughWithoutRetry.cs deleted file mode 100644 index 8680722dae04..000000000000 --- a/dotnet/src/SemanticKernel/Reliability/PassThroughWithoutRetry.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.Diagnostics; - -namespace Microsoft.SemanticKernel.Reliability; - -/// -/// A retry mechanism that does not retry. -/// -internal class PassThroughWithoutRetry : IRetryMechanism -{ - public async Task ExecuteWithRetryAsync(Func action, ILogger log, CancellationToken cancellationToken = default) - { - try - { - if (!cancellationToken.IsCancellationRequested) - { - await action(); - } - } - catch (Exception ex) when (!ex.IsCriticalException()) - { - log.LogWarning(ex, "Error executing action, not retrying"); - throw; - } - } -} diff --git a/dotnet/src/SemanticKernel/SemanticKernel.csproj b/dotnet/src/SemanticKernel/SemanticKernel.csproj index 6d8cc6ba880c..0ae7415b28a3 100644 --- a/dotnet/src/SemanticKernel/SemanticKernel.csproj +++ b/dotnet/src/SemanticKernel/SemanticKernel.csproj @@ -39,5 +39,8 @@ <_Parameter1>SemanticKernel.UnitTests + + <_Parameter1>DynamicProxyGenAssembly2 + \ No newline at end of file diff --git a/samples/dotnet/KernelBuilder/Program.cs b/samples/dotnet/KernelBuilder/Program.cs index c542a2dfe811..5f0e6c6d775a 100644 --- a/samples/dotnet/KernelBuilder/Program.cs +++ b/samples/dotnet/KernelBuilder/Program.cs @@ -99,20 +99,54 @@ .SetDefaultEmbeddingsBackend("myName3"); // ========================================================================================================== -// When invoking AI, by default the kernel will not retry on transient errors, such as throttling -// and timeouts. This behavior can be customized injecting a retry strategy that applies to all +// When invoking AI, by default the kernel will retry on transient errors, such as throttling and timeouts. +// The default behavior can be configured or a custom retry handler can be injected that will apply to all // AI requests (when using the kernel). var kernel8 = Kernel.Builder - .Configure(c => c.SetRetryMechanism(new RetryThreeTimes())) + .Configure(c => c.SetDefaultHttpRetryConfig(new HttpRetryConfig + { + MaxRetryCount = 3, + UseExponentialBackoff = true, + // MinRetryDelay = TimeSpan.FromSeconds(2), + // MaxRetryDelay = TimeSpan.FromSeconds(8), + // MaxTotalRetryTime = TimeSpan.FromSeconds(30), + // RetryableStatusCodes = new[] { HttpStatusCode.TooManyRequests, HttpStatusCode.RequestTimeout }, + // RetryableExceptions = new[] { typeof(HttpRequestException) } + })) + .Build(); + +var kernel9 = Kernel.Builder + .Configure(c => c.SetHttpRetryHandlerFactory(new NullHttpRetryHandlerFactory())) .Build(); -public class RetryThreeTimes : IRetryMechanism +var kernel10 = Kernel.Builder.WithRetryHandlerFactory(new RetryThreeTimesFactory()).Build(); + +// Example of a basic custom retry handler +public class RetryThreeTimesFactory : IDelegatingHandlerFactory { - public Task ExecuteWithRetryAsync(Func action, ILogger log, CancellationToken cancellationToken = default) + public DelegatingHandler Create(ILogger log) + { + return new RetryThreeTimes(log); + } +} + +public class RetryThreeTimes : DelegatingHandler +{ + private readonly AsyncRetryPolicy _policy; + + public RetryThreeTimes(ILogger log = null) + { + this._policy = GetPolicy(log ?? NullLogger.Instance); + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { - var policy = GetPolicy(log); - return policy.ExecuteAsync((_) => action(), cancellationToken); + return await this._policy.ExecuteAsync(async () => + { + var response = await base.SendAsync(request, cancellationToken); + return response; + }); } private static AsyncRetryPolicy GetPolicy(ILogger log) @@ -126,7 +160,7 @@ private static AsyncRetryPolicy GetPolicy(ILogger log) TimeSpan.FromSeconds(8) }, (ex, timespan, retryCount, _) => log.LogWarning(ex, - "Error executing action [attempt {0} of ], pausing {1} msecs", + "Error executing action [attempt {0} of 3], pausing {1}ms", retryCount, timespan.TotalMilliseconds)); } } diff --git a/samples/dotnet/kernel-syntax-examples/Example08_RetryHandler.cs b/samples/dotnet/kernel-syntax-examples/Example08_RetryHandler.cs new file mode 100644 index 000000000000..acbbdc316372 --- /dev/null +++ b/samples/dotnet/kernel-syntax-examples/Example08_RetryHandler.cs @@ -0,0 +1,174 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.CoreSkills; +using Microsoft.SemanticKernel.KernelExtensions; +using Microsoft.SemanticKernel.Reliability; +using Reliability; +using RepoUtils; + +// ReSharper disable once InconsistentNaming +public static class Example08_RetryHandler +{ + public static async Task RunAsync() + { + var kernel = InitializeKernel(); + var retryHandlerFactory = new RetryThreeTimesWithBackoffFactory(); + ConsoleLogger.Log.LogInformation("============================== RetryThreeTimesWithBackoff =============================="); + await RunRetryPolicyAsync(kernel, retryHandlerFactory); + + ConsoleLogger.Log.LogInformation("========================= RetryThreeTimesWithRetryAfterBackoff ========================="); + await RunRetryPolicyBuilderAsync(typeof(RetryThreeTimesWithRetryAfterBackoffFactory)); + + ConsoleLogger.Log.LogInformation("==================================== NoRetryPolicy ====================================="); + await RunRetryPolicyBuilderAsync(typeof(NullHttpRetryHandlerFactory)); + + ConsoleLogger.Log.LogInformation("=============================== DefaultHttpRetryHandler ================================"); + await RunRetryHandlerConfigAsync(new HttpRetryConfig() { MaxRetryCount = 3, UseExponentialBackoff = true }); + + ConsoleLogger.Log.LogInformation("======= DefaultHttpRetryConfig [MaxRetryCount = 3, UseExponentialBackoff = true] ======="); + await RunRetryHandlerConfigAsync(new HttpRetryConfig() { MaxRetryCount = 3, UseExponentialBackoff = true }); + } + + private static async Task RunRetryHandlerConfigAsync(HttpRetryConfig? config = null) + { + var kernelBuilder = Kernel.Builder.WithLogger(ConsoleLogger.Log); + if (config != null) + { + kernelBuilder = kernelBuilder.Configure(c => c.SetDefaultHttpRetryConfig(config)); + } + + // Add 401 to the list of retryable status codes + // Typically 401 would not be something we retry but for demonstration + // purposes we are doing so as it's easy to trigger when using an invalid key. + kernelBuilder = kernelBuilder.Configure(c => c.DefaultHttpRetryConfig.RetryableStatusCodes.Add(System.Net.HttpStatusCode.Unauthorized)); + + // OpenAI settings - you can set the OPENAI_API_KEY to an invalid value to see the retry policy in play + kernelBuilder = kernelBuilder.Configure(c => c.AddOpenAICompletionBackend("text-davinci-003", "text-davinci-003", "BAD_KEY")); + + var kernel = kernelBuilder.Build(); + + await ImportAndExecuteSkillAsync(kernel); + } + + private static IKernel InitializeKernel() + { + var kernel = Kernel.Builder.WithLogger(ConsoleLogger.Log).Build(); + // OpenAI settings - you can set the OPENAI_API_KEY to an invalid value to see the retry policy in play + kernel.Config.AddOpenAICompletionBackend("text-davinci-003", "text-davinci-003", "BAD_KEY"); + + return kernel; + } + + private static async Task RunRetryPolicyAsync(IKernel kernel, IDelegatingHandlerFactory retryHandlerFactory) + { + kernel.Config.SetHttpRetryHandlerFactory(retryHandlerFactory); + await ImportAndExecuteSkillAsync(kernel); + } + + private static async Task RunRetryPolicyBuilderAsync(Type retryHandlerFactoryType) + { + var kernelBuilder = Kernel.Builder.WithLogger(ConsoleLogger.Log) + .WithRetryHandlerFactory((Activator.CreateInstance(retryHandlerFactoryType) as IDelegatingHandlerFactory)!); + + // OpenAI settings - you can set the OPENAI_API_KEY to an invalid value to see the retry policy in play + kernelBuilder = kernelBuilder.Configure(c => c.AddOpenAICompletionBackend("text-davinci-003", "text-davinci-003", "BAD_KEY")); + + var kernel = kernelBuilder.Build(); + + await ImportAndExecuteSkillAsync(kernel); + } + + private static async Task ImportAndExecuteSkillAsync(IKernel kernel) + { + // Load semantic skill defined with prompt templates + string folder = RepoFiles.SampleSkillsPath(); + + kernel.ImportSkill(new TimeSkill(), "time"); + + var qaSkill = kernel.ImportSemanticSkillFromDirectory( + folder, + "QASkill"); + + var question = "How popular is Polly library?"; + + ConsoleLogger.Log.LogInformation("Question: {0}", question); + // To see the retry policy in play, you can set the OPENAI_API_KEY to an invalid value + var answer = await kernel.RunAsync(question, qaSkill["Question"]); + ConsoleLogger.Log.LogInformation("Answer: {0}", answer); + } +} + +/* Output: +info: object[0] + ============================== RetryThreeTimesWithBackoff ============================== +info: object[0] + Question: How popular is Polly library? +warn: object[0] + Error executing action [attempt 1 of 3], pausing 2000ms. Outcome: Unauthorized +warn: object[0] + Error executing action [attempt 2 of 3], pausing 4000ms. Outcome: Unauthorized +warn: object[0] + Error executing action [attempt 3 of 3], pausing 8000ms. Outcome: Unauthorized +fail: object[0] + Function call fail during pipeline step 0: QASkill.Question +info: object[0] + Answer: Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized +info: object[0] + ========================= RetryThreeTimesWithRetryAfterBackoff ========================= +info: object[0] + Question: How popular is Polly library? +warn: object[0] + Error executing action [attempt 1 of 3], pausing 2000ms. Outcome: Unauthorized +warn: object[0] + Error executing action [attempt 2 of 3], pausing 2000ms. Outcome: Unauthorized +warn: object[0] + Error executing action [attempt 3 of 3], pausing 2000ms. Outcome: Unauthorized +fail: object[0] + Function call fail during pipeline step 0: QASkill.Question +info: object[0] + Answer: Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized +info: object[0] + ==================================== NoRetryPolicy ===================================== +info: object[0] + Question: How popular is Polly library? +fail: object[0] + Function call fail during pipeline step 0: QASkill.Question +info: object[0] + Answer: Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized +info: object[0] + =============================== DefaultHttpRetryHandler ================================ +info: object[0] + Question: How popular is Polly library? +warn: object[0] + Error executing action [attempt 1 of 3]. Reason: Unauthorized. Will retry after 2000ms +warn: object[0] + Error executing action [attempt 2 of 3]. Reason: Unauthorized. Will retry after 4000ms +warn: object[0] + Error executing action [attempt 3 of 3]. Reason: Unauthorized. Will retry after 8000ms +fail: object[0] + Error executing request, max retry count reached. Reason: Unauthorized +fail: object[0] + Function call fail during pipeline step 0: QASkill.Question +info: object[0] + Answer: Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized +info: object[0] + ======= DefaultHttpRetryConfig [MaxRetryCount = 3, UseExponentialBackoff = true] ======= +info: object[0] + Question: How popular is Polly library? +warn: object[0] + Error executing action [attempt 1 of 3]. Reason: Unauthorized. Will retry after 2000ms +warn: object[0] + Error executing action [attempt 2 of 3]. Reason: Unauthorized. Will retry after 4000ms +warn: object[0] + Error executing action [attempt 3 of 3]. Reason: Unauthorized. Will retry after 8000ms +fail: object[0] + Error executing request, max retry count reached. Reason: Unauthorized +fail: object[0] + Function call fail during pipeline step 0: QASkill.Question +info: object[0] + Answer: Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized +*/ diff --git a/samples/dotnet/kernel-syntax-examples/Example08_RetryMechanism.cs b/samples/dotnet/kernel-syntax-examples/Example08_RetryMechanism.cs deleted file mode 100644 index 115d87b62e6a..000000000000 --- a/samples/dotnet/kernel-syntax-examples/Example08_RetryMechanism.cs +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.CoreSkills; -using Microsoft.SemanticKernel.KernelExtensions; -using Reliability; -using RepoUtils; - -// ReSharper disable once InconsistentNaming -public static class Example08_RetryMechanism -{ - public static async Task RunAsync() - { - Console.WriteLine("============================ RetryMechanism ============================"); - - IKernel kernel = Kernel.Builder.WithLogger(ConsoleLogger.Log).Build(); - kernel.Config.SetRetryMechanism(new RetryThreeTimesWithBackoff()); - - // OpenAI settings - kernel.Config.AddOpenAICompletionBackend("text-davinci-003", "text-davinci-003", Env.Var("OPENAI_API_KEY")); - - // Load semantic skill defined with prompt templates - string folder = RepoFiles.SampleSkillsPath(); - - kernel.ImportSkill(new TimeSkill(), "time"); - - var qaSkill = kernel.ImportSemanticSkillFromDirectory( - folder, - "QASkill"); - - var question = "How popular is Polly library?"; - var answer = await kernel.RunAsync(question, qaSkill["Question"]); - - Console.WriteLine($"Question: {question}\n\n" + answer); - } -} diff --git a/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs b/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs index f0038055c222..6bc930db92d3 100644 --- a/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs +++ b/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs @@ -9,7 +9,6 @@ using Microsoft.SemanticKernel.KernelExtensions; using Microsoft.SemanticKernel.Orchestration; using Microsoft.SemanticKernel.Orchestration.Extensions; -using Reliability; using RepoUtils; using Skills; @@ -46,7 +45,7 @@ private static async Task PoetrySamplesAsync() Console.WriteLine("Original plan:"); Console.WriteLine(originalPlan.Variables.ToPlan().PlanString); - _ = await ExecutePlanAsync(kernel, planner, originalPlan, 5); + await ExecutePlanAsync(kernel, planner, originalPlan, 5); } private static async Task EmailSamplesAsync() @@ -87,7 +86,7 @@ private static async Task EmailSamplesAsync() "and the kingdom was at peace once again. The king was so grateful to Mira that he asked her to marry him and she agreed. " + "They ruled the kingdom together, ruling with fairness and compassion, just as Arjun had done before. They lived " + "happily ever after, with the people of the kingdom remembering Mira as the brave young woman who saved them from the dragon."); - _ = await ExecutePlanAsync(kernel, planner, executionResults, 5); + await ExecutePlanAsync(kernel, planner, executionResults, 5); } private static async Task BookSamplesAsync() @@ -118,7 +117,7 @@ private static async Task BookSamplesAsync() Stopwatch sw = new(); sw.Start(); - _ = await ExecutePlanAsync(kernel, planner, originalPlan); + await ExecutePlanAsync(kernel, planner, originalPlan); } private static IKernel InitializeKernelAndPlanner(out IDictionary planner) @@ -129,7 +128,6 @@ private static IKernel InitializeKernelAndPlanner(out IDictionary -/// An example of a retry mechanism that retries three times with backoff. +/// A factory for creating a retry handler. /// -public class RetryThreeTimesWithBackoff : IRetryMechanism +public class RetryThreeTimesWithBackoffFactory : IDelegatingHandlerFactory { - public Task ExecuteWithRetryAsync(Func action, ILogger log, CancellationToken cancellationToken = default) + public DelegatingHandler Create(ILogger log) { - var policy = GetPolicy(log); - return policy.ExecuteAsync((_) => action(), cancellationToken); + return new RetryThreeTimesWithBackoff(log); + } +} + +/// +/// A basic example of a retry mechanism that retries three times with backoff. +/// +public class RetryThreeTimesWithBackoff : DelegatingHandler +{ + private readonly AsyncRetryPolicy _policy; + + public RetryThreeTimesWithBackoff(ILogger log) + { + this._policy = GetPolicy(log); + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + return await this._policy.ExecuteAsync(async () => + { + var response = await base.SendAsync(request, cancellationToken); + return response; + }); } - private static AsyncRetryPolicy GetPolicy(ILogger log) + private static AsyncRetryPolicy GetPolicy(ILogger log) { + // Handle 429 and 401 errors + // Typically 401 would not be something we retry but for demonstration + // purposes we are doing so as it's easy to trigger when using an invalid key. return Policy - .Handle(ex => ex.ErrorCode == AIException.ErrorCodes.Throttling) + .HandleResult(response => + response.StatusCode is System.Net.HttpStatusCode.TooManyRequests or System.Net.HttpStatusCode.Unauthorized) .WaitAndRetryAsync(new[] { TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(4), TimeSpan.FromSeconds(8) }, - (ex, timespan, retryCount, _) => log.LogWarning(ex, - "Error executing action [attempt {0} of ], pausing {1} msecs", retryCount, timespan.TotalMilliseconds)); + (outcome, timespan, retryCount, _) => log.LogWarning( + "Error executing action [attempt {0} of 3], pausing {1}ms. Outcome: {2}", + retryCount, + timespan.TotalMilliseconds, + outcome.Result.StatusCode)); } } diff --git a/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithRetryAfterBackoff.cs b/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithRetryAfterBackoff.cs new file mode 100644 index 000000000000..e55591ae88df --- /dev/null +++ b/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithRetryAfterBackoff.cs @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Reliability; +using Polly; +using Polly.Retry; + +namespace Reliability; + +/// +/// A factory for creating a retry handler. +/// +public class RetryThreeTimesWithRetryAfterBackoffFactory : IDelegatingHandlerFactory +{ + public DelegatingHandler Create(ILogger log) + { + return new RetryThreeTimesWithRetryAfterBackoff(log); + } +} + +/// +/// An example of a retry mechanism that retries three times with backoff using the RetryAfter value. +/// +public class RetryThreeTimesWithRetryAfterBackoff : DelegatingHandler +{ + private readonly AsyncRetryPolicy _policy; + + public RetryThreeTimesWithRetryAfterBackoff(ILogger log) + { + this._policy = GetPolicy(log); + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + return await this._policy.ExecuteAsync(async () => + { + var response = await base.SendAsync(request, cancellationToken); + return response; + }); + } + + private static AsyncRetryPolicy GetPolicy(ILogger log) + { + // Handle 429 and 401 errors + // Typically 401 would not be something we retry but for demonstration + // purposes we are doing so as it's easy to trigger when using an invalid key. + return Policy + .HandleResult(response => + response.StatusCode is System.Net.HttpStatusCode.TooManyRequests or System.Net.HttpStatusCode.Unauthorized) + .WaitAndRetryAsync( + retryCount: 3, + sleepDurationProvider: (_, r, _) => + { + var response = r.Result; + var retryAfter = response.Headers.RetryAfter?.Delta ?? response.Headers.RetryAfter?.Date - DateTimeOffset.Now; + return retryAfter ?? TimeSpan.FromSeconds(2); + }, + (outcome, timespan, retryCount, _) => + { + log.LogWarning( + "Error executing action [attempt {0} of 3], pausing {1}ms. Outcome: {2}", + retryCount, + timespan.TotalMilliseconds, + outcome.Result.StatusCode); + return Task.CompletedTask; + }); + } +} diff --git a/samples/dotnet/kernel-syntax-examples/RepoUtils/ConsoleLogger.cs b/samples/dotnet/kernel-syntax-examples/RepoUtils/ConsoleLogger.cs index 51aa7055668b..a3d7ff63a8f9 100644 --- a/samples/dotnet/kernel-syntax-examples/RepoUtils/ConsoleLogger.cs +++ b/samples/dotnet/kernel-syntax-examples/RepoUtils/ConsoleLogger.cs @@ -13,18 +13,20 @@ internal static class ConsoleLogger internal static ILogger Log => LogFactory.CreateLogger(); private static ILoggerFactory LogFactory => s_loggerFactory.Value; + private static readonly Lazy s_loggerFactory = new(LogBuilder); private static ILoggerFactory LogBuilder() { return LoggerFactory.Create(builder => { - builder.SetMinimumLevel(LogLevel.Warning); + // builder.SetMinimumLevel(LogLevel.Warning); // builder.AddFilter("Microsoft", LogLevel.Trace); // builder.AddFilter("Microsoft", LogLevel.Debug); // builder.AddFilter("Microsoft", LogLevel.Information); - // builder.AddFilter("Microsoft", LogLevel.Warning); + builder.AddFilter("Microsoft", LogLevel.Warning); // builder.AddFilter("Microsoft", LogLevel.Error); + builder.AddFilter("System", LogLevel.Warning); builder.AddConsole(); }); }