diff --git a/dotnet/src/IntegrationTest/AI/OpenAICompletionTests.cs b/dotnet/src/IntegrationTest/AI/OpenAICompletionTests.cs index b7cdadcf4c32..ec882f8a7cf9 100644 --- a/dotnet/src/IntegrationTest/AI/OpenAICompletionTests.cs +++ b/dotnet/src/IntegrationTest/AI/OpenAICompletionTests.cs @@ -8,6 +8,7 @@ using IntegrationTests.TestSettings; using Microsoft.Extensions.Configuration; using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Configuration; using Microsoft.SemanticKernel.KernelExtensions; using Microsoft.SemanticKernel.Orchestration; using Xunit; @@ -91,6 +92,35 @@ public async Task AzureOpenAITestAsync(string prompt, string expectedAnswerConta Assert.Contains(expectedAnswerContains, actual.Result, StringComparison.InvariantCultureIgnoreCase); } + [Theory] + [InlineData("Where is the most famous fish market in Seattle, Washington, USA?", + "Error executing action [attempt 1 of 1]. Reason: Unauthorized. Will retry after 2000ms")] + public async Task OpenAIHttpRetryPolicyTestAsync(string prompt, string expectedOutput) + { + // Arrange + var retryConfig = new KernelConfig.HttpRetryConfig(); + retryConfig.RetryableStatusCodes.Add(System.Net.HttpStatusCode.Unauthorized); + IKernel target = Kernel.Builder.WithLogger(this._testOutputHelper).Configure(c => c.SetDefaultHttpRetryConfig(retryConfig)).Build(); + + OpenAIConfiguration? openAIConfiguration = this._configuration.GetSection("OpenAI").Get(); + Assert.NotNull(openAIConfiguration); + + target.Config.AddOpenAICompletionBackend( + label: openAIConfiguration.Label, + modelId: openAIConfiguration.ModelId, + apiKey: "INVALID_KEY"); + + target.Config.SetDefaultCompletionBackend(openAIConfiguration.Label); + + IDictionary skill = GetSkill("SummarizeSkill", target); + + // Act + await target.RunAsync(prompt, skill["Summarize"]); + + // Assert + Assert.Contains(expectedOutput, this._testOutputHelper.GetLogs(), StringComparison.InvariantCultureIgnoreCase); + } + private static IDictionary GetSkill(string skillName, IKernel target) { string? currentAssemblyDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); diff --git a/dotnet/src/IntegrationTest/RedirectOutput.cs b/dotnet/src/IntegrationTest/RedirectOutput.cs index f3fadf13e8d2..173a1d36136d 100644 --- a/dotnet/src/IntegrationTest/RedirectOutput.cs +++ b/dotnet/src/IntegrationTest/RedirectOutput.cs @@ -1,18 +1,22 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.IO; using System.Text; +using Microsoft.Extensions.Logging; using Xunit.Abstractions; namespace IntegrationTests; -public class RedirectOutput : TextWriter +public class RedirectOutput : TextWriter, ILogger { private readonly ITestOutputHelper _output; + private readonly StringBuilder _logs; public RedirectOutput(ITestOutputHelper output) { this._output = output; + this._logs = new StringBuilder(); } public override Encoding Encoding { get; } = Encoding.UTF8; @@ -20,5 +24,28 @@ public RedirectOutput(ITestOutputHelper output) public override void WriteLine(string? value) { this._output.WriteLine(value); + this._logs.AppendLine(value); + } + + public IDisposable? BeginScope(TState state) where TState : notnull + { + return null; + } + + public bool IsEnabled(LogLevel logLevel) + { + return true; + } + + public string GetLogs() + { + return this._logs.ToString(); + } + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + var message = formatter(state, exception); + this._output?.WriteLine(message); + this._logs.AppendLine(message); } } diff --git a/dotnet/src/SemanticKernel.Test/Configuration/KernelConfigTests.cs b/dotnet/src/SemanticKernel.Test/Configuration/KernelConfigTests.cs index 3dc94106a6fb..3134586ab43d 100644 --- a/dotnet/src/SemanticKernel.Test/Configuration/KernelConfigTests.cs +++ b/dotnet/src/SemanticKernel.Test/Configuration/KernelConfigTests.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using System.Linq; +using System.Threading.Tasks; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.AI.OpenAI.Services; using Microsoft.SemanticKernel.Configuration; @@ -16,55 +17,118 @@ namespace SemanticKernelTests.Configuration; public class KernelConfigTests { [Fact] - public void RetryMechanismIsSet() + public void HttpRetryHandlerFactoryIsSet() { // Arrange - var retry = new PassThroughWithoutRetry(); + var retry = new NullHttpRetryHandlerFactory(); var config = new KernelConfig(); // Act - config.SetRetryMechanism(retry); + config.SetHttpRetryHandlerFactory(retry); // Assert - Assert.Equal(retry, config.RetryMechanism); + Assert.Equal(retry, config.HttpHandlerFactory); } [Fact] - public void RetryMechanismIsSetWithCustomImplementation() + public void HttpRetryHandlerFactoryIsSetWithCustomImplementation() { // Arrange - var retry = new Mock(); + var retry = new Mock(); var config = new KernelConfig(); // Act - config.SetRetryMechanism(retry.Object); + config.SetHttpRetryHandlerFactory(retry.Object); // Assert - Assert.Equal(retry.Object, config.RetryMechanism); + Assert.Equal(retry.Object, config.HttpHandlerFactory); } [Fact] - public void RetryMechanismIsSetToPassThroughWithoutRetryIfNull() + public void HttpRetryHandlerFactoryIsSetToDefaultHttpRetryHandlerFactoryIfNull() { // Arrange var config = new KernelConfig(); // Act - config.SetRetryMechanism(null); + config.SetHttpRetryHandlerFactory(null); // Assert - Assert.IsType(config.RetryMechanism); + Assert.IsType(config.HttpHandlerFactory); } [Fact] - public void RetryMechanismIsSetToPassThroughWithoutRetryIfNotSet() + public void HttpRetryHandlerFactoryIsSetToDefaultHttpRetryHandlerFactoryIfNotSet() { // Arrange var config = new KernelConfig(); // Act // Assert - Assert.IsType(config.RetryMechanism); + Assert.IsType(config.HttpHandlerFactory); + } + + [Fact] + public async Task NegativeMaxRetryCountThrowsAsync() + { + // Act + await Assert.ThrowsAsync(() => + { + var httpRetryConfig = new KernelConfig.HttpRetryConfig() { MaxRetryCount = -1 }; + return Task.CompletedTask; + }); + } + + [Fact] + public void SetDefaultHttpRetryConfig() + { + // Arrange + var config = new KernelConfig(); + var httpRetryConfig = new KernelConfig.HttpRetryConfig() { MaxRetryCount = 1 }; + + // Act + config.SetDefaultHttpRetryConfig(httpRetryConfig); + + // Assert + Assert.Equal(httpRetryConfig, config.DefaultHttpRetryConfig); + } + + [Fact] + public void SetDefaultHttpRetryConfigToDefaultIfNotSet() + { + // Arrange + var config = new KernelConfig(); + + // Act + // Assert + var defaultConfig = new KernelConfig.HttpRetryConfig(); + Assert.Equal(defaultConfig.MaxRetryCount, config.DefaultHttpRetryConfig.MaxRetryCount); + Assert.Equal(defaultConfig.MaxRetryDelay, config.DefaultHttpRetryConfig.MaxRetryDelay); + Assert.Equal(defaultConfig.MinRetryDelay, config.DefaultHttpRetryConfig.MinRetryDelay); + Assert.Equal(defaultConfig.MaxTotalRetryTime, config.DefaultHttpRetryConfig.MaxTotalRetryTime); + Assert.Equal(defaultConfig.UseExponentialBackoff, config.DefaultHttpRetryConfig.UseExponentialBackoff); + Assert.Equal(defaultConfig.RetryableStatusCodes, config.DefaultHttpRetryConfig.RetryableStatusCodes); + Assert.Equal(defaultConfig.RetryableExceptionTypes, config.DefaultHttpRetryConfig.RetryableExceptionTypes); + } + + [Fact] + public void SetDefaultHttpRetryConfigToDefaultIfNull() + { + // Arrange + var config = new KernelConfig(); + + // Act + config.SetDefaultHttpRetryConfig(null); + + // Assert + var defaultConfig = new KernelConfig.HttpRetryConfig(); + Assert.Equal(defaultConfig.MaxRetryCount, config.DefaultHttpRetryConfig.MaxRetryCount); + Assert.Equal(defaultConfig.MaxRetryDelay, config.DefaultHttpRetryConfig.MaxRetryDelay); + Assert.Equal(defaultConfig.MinRetryDelay, config.DefaultHttpRetryConfig.MinRetryDelay); + Assert.Equal(defaultConfig.MaxTotalRetryTime, config.DefaultHttpRetryConfig.MaxTotalRetryTime); + Assert.Equal(defaultConfig.UseExponentialBackoff, config.DefaultHttpRetryConfig.UseExponentialBackoff); + Assert.Equal(defaultConfig.RetryableStatusCodes, config.DefaultHttpRetryConfig.RetryableStatusCodes); + Assert.Equal(defaultConfig.RetryableExceptionTypes, config.DefaultHttpRetryConfig.RetryableExceptionTypes); } [Fact] diff --git a/dotnet/src/SemanticKernel.Test/Reliability/DefaultHttpRetryHandlerTests.cs b/dotnet/src/SemanticKernel.Test/Reliability/DefaultHttpRetryHandlerTests.cs new file mode 100644 index 000000000000..5a051dccbcd1 --- /dev/null +++ b/dotnet/src/SemanticKernel.Test/Reliability/DefaultHttpRetryHandlerTests.cs @@ -0,0 +1,634 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Reliability; +using Moq; +using Moq.Protected; +using Xunit; +using static Microsoft.SemanticKernel.Configuration.KernelConfig; + +namespace SemanticKernelTests.Reliability; + +public class DefaultHttpRetryHandlerTests +{ + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + [InlineData(HttpStatusCode.TooManyRequests)] + public async Task NoMaxRetryCountCallsOnceForStatusAsync(HttpStatusCode statusCode) + { + // Arrange + using var retry = new DefaultHttpRetryHandler(new HttpRetryConfig() { MaxRetryCount = 0 }, Mock.Of()); + using var mockResponse = new HttpResponseMessage(statusCode); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + [InlineData(HttpStatusCode.TooManyRequests)] + public async Task ItRetriesOnceOnRetryableStatusAsync(HttpStatusCode statusCode) + { + // Arrange + using var retry = ConfigureRetryHandler(); + using var mockResponse = new HttpResponseMessage(statusCode); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + } + + [Theory] + [InlineData(typeof(HttpRequestException))] + public async Task ItRetriesOnceOnRetryableExceptionAsync(Type exceptionType) + { + // Arrange + using var retry = ConfigureRetryHandler(); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(exceptionType); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await Assert.ThrowsAsync(exceptionType, + async () => await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Theory] + [InlineData(typeof(HttpRequestException))] + public async Task NoMaxRetryCountCallsOnceForExceptionAsync(Type exceptionType) + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 0 }); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(exceptionType); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await Assert.ThrowsAsync(exceptionType, + async () => await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + [InlineData(HttpStatusCode.TooManyRequests)] + public async Task ItRetriesOnceOnTransientStatusWithExponentialBackoffAsync(HttpStatusCode statusCode) + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { UseExponentialBackoff = true }); + using var mockResponse = new HttpResponseMessage(statusCode); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + } + + [Theory] + [InlineData(typeof(HttpRequestException))] + public async Task ItRetriesOnceOnRetryableExceptionWithExponentialBackoffAsync(Type exceptionType) + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { UseExponentialBackoff = true }); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(exceptionType); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await Assert.ThrowsAsync(exceptionType, + async () => await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + public async Task ItRetriesExponentiallyWithExponentialBackoffAsync(HttpStatusCode statusCode) + { + // Arrange + var currentTime = DateTimeOffset.UtcNow; + var mockTimeProvider = new Mock(); + var mockDelayProvider = new Mock(); + mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) + .Returns(() => currentTime) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(5)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(510)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(1015)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(1520)); + using var retry = ConfigureRetryHandler(new HttpRetryConfig() + { + UseExponentialBackoff = true, MaxRetryCount = 3, + MinRetryDelay = TimeSpan.FromMilliseconds(500) + }, mockTimeProvider, mockDelayProvider); + using var mockResponse = new HttpResponseMessage(statusCode); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(4), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(4)); + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(500), It.IsAny()), Times.Once); + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(1000), It.IsAny()), Times.Once); + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(2000), It.IsAny()), Times.Once); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + public async Task ItRetriesOnceOnTransientStatusCodeWithRetryValueAsync(HttpStatusCode statusCode) + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig(), null); + using var mockResponse = new HttpResponseMessage() + { + StatusCode = statusCode, + Headers = { RetryAfter = new RetryConditionHeaderValue(new TimeSpan(0, 0, 0, 1)) }, + }; + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + using var testContent = new StringContent("test"); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + Assert.Equal(new TimeSpan(0, 0, 0, 1), response.Headers.RetryAfter?.Delta); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + public async Task ItRetriesStatusCustomCountAsync(HttpStatusCode expectedStatus) + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 3 }, null); + using var mockResponse = new HttpResponseMessage(expectedStatus); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(4), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(expectedStatus, response.StatusCode); + } + + [Theory] + [InlineData(typeof(HttpRequestException))] + public async Task ItRetriesExceptionsCustomCountAsync(Type expectedException) + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 3 }, null); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(expectedException); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await Assert.ThrowsAsync(expectedException, + async () => await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(4), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Fact] + public async Task NoExceptionNoRetryAsync() + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 3 }, null); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.OK); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(1), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + } + + [Fact] + public async Task ItDoesNotExecuteOnCancellationTokenAsync() + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 3 }, null); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.OK); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + var cancellationToken = new CancellationToken(true); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await Assert.ThrowsAsync(async () => + await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, cancellationToken)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Never(), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Fact] + public async Task ItDoestExecuteOnFalseCancellationTokenAsync() + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 3 }, null); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.OK); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + var cancellationToken = new CancellationToken(false); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, cancellationToken); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(1), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + } + + [Fact] + public async Task ItRetriesWithMinRetryDelayAsync() + { + var HttpRetryConfig = new HttpRetryConfig + { + MinRetryDelay = TimeSpan.FromMilliseconds(500) + }; + + var mockDelayProvider = new Mock(); + var mockTimeProvider = new Mock(); + + var currentTime = DateTimeOffset.UtcNow; + + mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) + .Returns(() => currentTime) + .Returns(() => currentTime.AddMilliseconds(5)) + .Returns(() => currentTime.AddMilliseconds(510)); + + mockDelayProvider.Setup(x => x.DelayAsync(It.IsAny(), It.IsAny())) + .Returns(() => Task.CompletedTask); + + using var retry = ConfigureRetryHandler(HttpRetryConfig, mockTimeProvider, mockDelayProvider); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(2)); + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(500), It.IsAny()), Times.Once); + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + } + + [Fact] + public async Task ItRetriesWithMaxRetryDelayAsync() + { + var HttpRetryConfig = new HttpRetryConfig + { + MinRetryDelay = TimeSpan.FromMilliseconds(1), + MaxRetryDelay = TimeSpan.FromMilliseconds(500) + }; + + var mockDelayProvider = new Mock(); + var mockTimeProvider = new Mock(); + + var currentTime = DateTimeOffset.UtcNow; + + mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) + .Returns(() => currentTime) + .Returns(() => currentTime.AddMilliseconds(5)) + .Returns(() => currentTime.AddMilliseconds(505)); + + mockDelayProvider.Setup(x => x.DelayAsync(It.IsAny(), It.IsAny())) + .Returns(() => Task.CompletedTask); + + using var retry = ConfigureRetryHandler(HttpRetryConfig, mockTimeProvider, mockDelayProvider); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests) + { Headers = { RetryAfter = new RetryConditionHeaderValue(TimeSpan.FromMilliseconds(2000)) } }; + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(2)); + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(500), It.IsAny()), Times.Once); + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + Assert.Equal(TimeSpan.FromMilliseconds(2000), response.Headers.RetryAfter?.Delta); + } + + [Theory] + [InlineData(HttpStatusCode.TooManyRequests)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + [InlineData(HttpStatusCode.RequestTimeout)] + public async Task ItRetriesWithMaxTotalDelayAsync(HttpStatusCode statusCode) + { + // Arrange + var HttpRetryConfig = new HttpRetryConfig + { + MaxRetryCount = 5, + MinRetryDelay = TimeSpan.FromMilliseconds(50), + MaxRetryDelay = TimeSpan.FromMilliseconds(50), + MaxTotalRetryTime = TimeSpan.FromMilliseconds(350) + }; + + var mockDelayProvider = new Mock(); + var mockTimeProvider = new Mock(); + + var currentTime = DateTimeOffset.UtcNow; + mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) + .Returns(() => currentTime) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(5)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(55)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(110)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(165)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(220)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(275)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(330)); + + using var retry = ConfigureRetryHandler(HttpRetryConfig, mockTimeProvider, mockDelayProvider); + + using var mockResponse = new HttpResponseMessage(statusCode); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(6)); // one for the initial call, and one for each of 5 attempts + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(50), It.IsAny()), Times.Exactly(5)); + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(6), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + } + + [Fact] + public async Task ItRetriesFewerWithMaxTotalDelayAsync() + { + // Arrange + var HttpRetryConfig = new HttpRetryConfig + { + MaxRetryCount = 5, + MinRetryDelay = TimeSpan.FromMilliseconds(50), + MaxRetryDelay = TimeSpan.FromMilliseconds(50), + MaxTotalRetryTime = TimeSpan.FromMilliseconds(100) + }; + + var mockDelayProvider = new Mock(); + var mockTimeProvider = new Mock(); + + var currentTime = DateTimeOffset.UtcNow; + mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) + .Returns(() => currentTime) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(5)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(55)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(110)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(165)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(220)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(275)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(330)); + + using var retry = ConfigureRetryHandler(HttpRetryConfig, mockTimeProvider, mockDelayProvider); + + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(3)); // one for the initial call, and one for each of 5 attempts + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(50), It.IsAny()), Times.Exactly(1)); + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + } + + [Fact] + public async Task ItRetriesOnRetryableStatusCodesAsync() + { + // Arrange + var config = new HttpRetryConfig() { RetryableStatusCodes = new List { HttpStatusCode.Unauthorized } }; + using var retry = ConfigureRetryHandler(config); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.Unauthorized); + + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); + } + + [Fact] + public async Task ItDoesNotRetryOnNonRetryableStatusCodesAsync() + { + // Arrange + var config = new HttpRetryConfig() { RetryableStatusCodes = new List { HttpStatusCode.Unauthorized } }; + using var retry = ConfigureRetryHandler(config); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); + + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + } + + [Fact] + public async Task ItRetriesOnRetryableExceptionsAsync() + { + // Arrange + var config = new HttpRetryConfig() { RetryableExceptionTypes = new List { typeof(InvalidOperationException) } }; + using var retry = ConfigureRetryHandler(config); + + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(typeof(InvalidOperationException)); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + await Assert.ThrowsAsync(async () => + await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Fact] + public async Task ItDoesNotRetryOnNonRetryableExceptionsAsync() + { + // Arrange + var config = new HttpRetryConfig() { RetryableExceptionTypes = new List { typeof(InvalidOperationException) } }; + using var retry = ConfigureRetryHandler(config); + + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(typeof(ArgumentException)); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + await Assert.ThrowsAsync(async () => + await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + } + + private static DefaultHttpRetryHandler ConfigureRetryHandler(HttpRetryConfig? config = null, + Mock? timeProvider = null, Mock? delayProvider = null) + { + delayProvider ??= new Mock(); + timeProvider ??= new Mock(); + var retry = new DefaultHttpRetryHandler(config ?? new HttpRetryConfig(), Mock.Of(), delayProvider.Object, timeProvider.Object); + return retry; + } + + private static Mock GetHttpMessageHandlerMock(HttpResponseMessage mockResponse) + { + var mockHandler = new Mock(); + mockHandler.Protected() + .Setup>("SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .ReturnsAsync(mockResponse); + return mockHandler; + } + + private static Mock GetHttpMessageHandlerMock(Type exceptionType) + { + var mockHandler = new Mock(); + mockHandler.Protected() + .Setup>("SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .ThrowsAsync(Activator.CreateInstance(exceptionType) as Exception); + return mockHandler; + } +} diff --git a/dotnet/src/SemanticKernel.Test/Reliability/NullHttpRetryHandlerTests.cs b/dotnet/src/SemanticKernel.Test/Reliability/NullHttpRetryHandlerTests.cs new file mode 100644 index 000000000000..8032700e13f7 --- /dev/null +++ b/dotnet/src/SemanticKernel.Test/Reliability/NullHttpRetryHandlerTests.cs @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Reliability; +using Moq; +using Moq.Protected; +using Xunit; + +namespace SemanticKernelTests.Reliability; + +public class NullHttpRetryHandlerTests +{ + [Fact] + public async Task ItDoesNotRetryOnExceptionAsync() + { + // Arrange + using var retry = new NullHttpRetryHandler(); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + } + + [Fact] + public async Task NoExceptionNoRetryAsync() + { + // Arrange + using var retry = new NullHttpRetryHandler(); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.OK); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + } + + [Fact] + public async Task TaskCanceledExceptionThrownOnCancellationTokenAsync() + { + // Arrange + using var retry = new NullHttpRetryHandler(); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + using var cancellationTokenSource = new CancellationTokenSource(); + cancellationTokenSource.Cancel(); + + // Act + await Assert.ThrowsAsync(async () => + await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, cancellationTokenSource.Token)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Fact] + public async Task ItDoestExecuteOnFalseCancellationTokenAsync() + { + // Arrange + using var retry = new NullHttpRetryHandler(); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, new CancellationToken(false)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + } + + private static Mock GetHttpMessageHandlerMock(HttpResponseMessage mockResponse) + { + var mockHandler = new Mock(); + mockHandler.Protected() + .Setup>("SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .ReturnsAsync(mockResponse); + return mockHandler; + } +} diff --git a/dotnet/src/SemanticKernel.Test/Reliability/PassThroughWithoutRetryTests.cs b/dotnet/src/SemanticKernel.Test/Reliability/PassThroughWithoutRetryTests.cs deleted file mode 100644 index 85f905d70f77..000000000000 --- a/dotnet/src/SemanticKernel.Test/Reliability/PassThroughWithoutRetryTests.cs +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.AI; -using Microsoft.SemanticKernel.Reliability; -using Moq; -using Xunit; - -namespace SemanticKernelTests.Reliability; - -public class PassThroughWithoutRetryTests -{ - [Fact] - public async Task ItDoesNotRetryOnExceptionAsync() - { - // Arrange - var retry = new PassThroughWithoutRetry(); - var action = new Mock>(); - action.Setup(a => a()).Throws(new AIException(AIException.ErrorCodes.Throttling, "Throttling Test")); - - // Act - await Assert.ThrowsAsync(() => retry.ExecuteWithRetryAsync(action.Object, Mock.Of())); - - // Assert - action.Verify(a => a(), Times.Once); - } - - [Fact] - public async Task NoExceptionNoRetryAsync() - { - // Arrange - var log = new Mock(); - var retry = new PassThroughWithoutRetry(); - var action = new Mock>(); - - // Act - await retry.ExecuteWithRetryAsync(action.Object, log.Object); - - // Assert - action.Verify(a => a(), Times.Once); - } - - [Fact] - public async Task ItDoesNotExecuteOnCancellationTokenAsync() - { - // Arrange - var retry = new PassThroughWithoutRetry(); - var action = new Mock>(); - action.Setup(a => a()).Throws(new AIException(AIException.ErrorCodes.Throttling, "Throttling Test")); - - // Act - await retry.ExecuteWithRetryAsync(action.Object, Mock.Of(), new CancellationToken(true)); - - // Assert - action.Verify(a => a(), Times.Never); - } - - [Fact] - public async Task ItDoestExecuteOnFalseCancellationTokenAsync() - { - // Arrange - var retry = new PassThroughWithoutRetry(); - var action = new Mock>(); - action.Setup(a => a()).Throws(new AIException(AIException.ErrorCodes.Throttling, "Throttling Test")); - - // Act - await Assert.ThrowsAsync(() => retry.ExecuteWithRetryAsync(action.Object, Mock.Of(), new CancellationToken(false))); - - // Assert - action.Verify(a => a(), Times.Once); - } -} diff --git a/dotnet/src/SemanticKernel/AI/ITextCompletionClient.cs b/dotnet/src/SemanticKernel/AI/ITextCompletionClient.cs index a4118375d8db..833c6b495c6f 100644 --- a/dotnet/src/SemanticKernel/AI/ITextCompletionClient.cs +++ b/dotnet/src/SemanticKernel/AI/ITextCompletionClient.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Threading; using System.Threading.Tasks; namespace Microsoft.SemanticKernel.AI; @@ -14,6 +15,7 @@ public interface ITextCompletionClient /// /// The prompt to complete. /// Request settings for the completion API + /// Cancellation token /// Text generated by the remote model - public Task CompleteAsync(string text, CompleteRequestSettings requestSettings); + public Task CompleteAsync(string text, CompleteRequestSettings requestSettings, CancellationToken cancellationToken = default); } diff --git a/dotnet/src/SemanticKernel/AI/OpenAI/Clients/AzureOpenAIClientAbstract.cs b/dotnet/src/SemanticKernel/AI/OpenAI/Clients/AzureOpenAIClientAbstract.cs index f3ab80244db7..3ac693ed051f 100644 --- a/dotnet/src/SemanticKernel/AI/OpenAI/Clients/AzureOpenAIClientAbstract.cs +++ b/dotnet/src/SemanticKernel/AI/OpenAI/Clients/AzureOpenAIClientAbstract.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel.AI.OpenAI.HttpSchema; +using Microsoft.SemanticKernel.Reliability; using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.AI.OpenAI.Clients; @@ -25,7 +26,7 @@ public abstract class AzureOpenAIClientAbstract : OpenAIClientAbstract /// protected string AzureOpenAIApiVersion { - get { return this._azureOpenAIApiVersion; } + get => this._azureOpenAIApiVersion; set { if (string.IsNullOrWhiteSpace(value)) @@ -48,7 +49,8 @@ protected string AzureOpenAIApiVersion /// Construct an AzureOpenAIClientAbstract object /// /// Logger - protected AzureOpenAIClientAbstract(ILogger? log = null) : base(log) + /// Retry handler factory + protected AzureOpenAIClientAbstract(ILogger? log = null, IDelegatingHandlerFactory? handlerFactory = null) : base(log, handlerFactory) { } @@ -161,5 +163,5 @@ protected async Task CacheDeploymentsAsync() private string _azureOpenAIApiVersion = DefaultAzureAPIVersion; - #endregion + #endregion private ================================================================================ } diff --git a/dotnet/src/SemanticKernel/AI/OpenAI/Clients/OpenAIClientAbstract.cs b/dotnet/src/SemanticKernel/AI/OpenAI/Clients/OpenAIClientAbstract.cs index 869cb9de2e41..022f35ce15f0 100644 --- a/dotnet/src/SemanticKernel/AI/OpenAI/Clients/OpenAIClientAbstract.cs +++ b/dotnet/src/SemanticKernel/AI/OpenAI/Clients/OpenAIClientAbstract.cs @@ -7,11 +7,13 @@ using System.Net.Http; using System.Net.Mime; using System.Text; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.SemanticKernel.AI.Embeddings; using Microsoft.SemanticKernel.AI.OpenAI.HttpSchema; +using Microsoft.SemanticKernel.Reliability; using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.AI.OpenAI.Clients; @@ -33,14 +35,19 @@ public abstract class OpenAIClientAbstract : IDisposable protected HttpClient HTTPClient { get; } private readonly HttpClientHandler _httpClientHandler; + private readonly IDelegatingHandlerFactory _handlerFactory = new DefaultHttpRetryHandlerFactory(); + private readonly DelegatingHandler _retryHandler; - internal OpenAIClientAbstract(ILogger? log = null) + internal OpenAIClientAbstract(ILogger? log = null, IDelegatingHandlerFactory? handlerFactory = null) { - if (log != null) { this.Log = log; } + this.Log = log ?? this.Log; + this._handlerFactory = handlerFactory ?? this._handlerFactory; - // TODO: allow injection of retry logic, e.g. Polly this._httpClientHandler = new() { CheckCertificateRevocationList = true }; - this.HTTPClient = new HttpClient(this._httpClientHandler); + this._retryHandler = this._handlerFactory.Create(this.Log); + this._retryHandler.InnerHandler = this._httpClientHandler; + + this.HTTPClient = new HttpClient(this._retryHandler); this.HTTPClient.DefaultRequestHeaders.Add("User-Agent", HTTPUseragent); } @@ -49,15 +56,16 @@ internal OpenAIClientAbstract(ILogger? log = null) /// /// URL for the completion request API /// Prompt to complete + /// Cancellation token /// The completed text /// AIException thrown during the request. - protected async Task ExecuteCompleteRequestAsync(string url, string requestBody) + protected async Task ExecuteCompleteRequestAsync(string url, string requestBody, CancellationToken cancellationToken = default) { try { this.Log.LogDebug("Sending completion request to {0}: {1}", url, requestBody); - var result = await this.ExecutePostRequestAsync(url, requestBody); + var result = await this.ExecutePostRequestAsync(url, requestBody, cancellationToken); if (result.Completions.Count < 1) { throw new AIException( @@ -124,6 +132,7 @@ protected virtual void Dispose(bool disposing) { this.HTTPClient.Dispose(); this._httpClientHandler.Dispose(); + this._retryHandler.Dispose(); } } @@ -132,14 +141,14 @@ protected virtual void Dispose(bool disposing) // HTTP user agent sent to remote endpoints private const string HTTPUseragent = "Microsoft Semantic Kernel"; - private async Task ExecutePostRequestAsync(string url, string requestBody) + private async Task ExecutePostRequestAsync(string url, string requestBody, CancellationToken cancellationToken = default) { string responseJson; try { using HttpContent content = new StringContent(requestBody, Encoding.UTF8, MediaTypeNames.Application.Json); - HttpResponseMessage response = await this.HTTPClient.PostAsync(url, content); + HttpResponseMessage response = await this.HTTPClient.PostAsync(url, content, cancellationToken); if (response == null) { diff --git a/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextCompletion.cs b/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextCompletion.cs index 38e8055a47c4..aa4cbff440d9 100644 --- a/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextCompletion.cs +++ b/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextCompletion.cs @@ -1,10 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel.AI.OpenAI.Clients; using Microsoft.SemanticKernel.AI.OpenAI.HttpSchema; using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Reliability; using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.AI.OpenAI.Services; @@ -22,8 +24,10 @@ public sealed class AzureTextCompletion : AzureOpenAIClientAbstract, ITextComple /// Azure OpenAI API key, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// Azure OpenAI API version, see https://learn.microsoft.com/azure/cognitive-services/openai/reference /// Application logger - public AzureTextCompletion(string modelId, string endpoint, string apiKey, string apiVersion, ILogger? log = null) - : base(log) + /// Retry handler factory for HTTP requests. + public AzureTextCompletion(string modelId, string endpoint, string apiKey, string apiVersion, ILogger? log = null, + IDelegatingHandlerFactory? handlerFactory = null) + : base(log, handlerFactory) { Verify.NotEmpty(modelId, "The ID cannot be empty, you must provide a Model ID or a Deployment name."); this._modelId = modelId; @@ -43,9 +47,10 @@ public AzureTextCompletion(string modelId, string endpoint, string apiKey, strin /// /// Text to complete /// Request settings for the completion API + /// Cancellation token /// The completed text. /// AIException thrown during the request - public async Task CompleteAsync(string text, CompleteRequestSettings requestSettings) + public async Task CompleteAsync(string text, CompleteRequestSettings requestSettings, CancellationToken cancellationToken = default) { Verify.NotNull(requestSettings, "Completion settings cannot be empty"); @@ -72,7 +77,7 @@ public async Task CompleteAsync(string text, CompleteRequestSettings req Stop = requestSettings.StopSequences is { Count: > 0 } ? requestSettings.StopSequences : null, }); - return await this.ExecuteCompleteRequestAsync(url, requestBody); + return await this.ExecuteCompleteRequestAsync(url, requestBody, cancellationToken); } #region private ================================================================================ diff --git a/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextEmbeddings.cs b/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextEmbeddings.cs index e7c2dfac1a51..283d56d5b3db 100644 --- a/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextEmbeddings.cs +++ b/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextEmbeddings.cs @@ -7,6 +7,7 @@ using Microsoft.SemanticKernel.AI.OpenAI.Clients; using Microsoft.SemanticKernel.AI.OpenAI.HttpSchema; using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Reliability; using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.AI.OpenAI.Services; @@ -26,8 +27,10 @@ public sealed class AzureTextEmbeddings : AzureOpenAIClientAbstract, IEmbeddingG /// Azure OpenAI API key, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// Azure OpenAI API version, see https://learn.microsoft.com/azure/cognitive-services/openai/reference /// Application logger - public AzureTextEmbeddings(string modelId, string endpoint, string apiKey, string apiVersion, ILogger? log = null) - : base(log) + /// An optional HTTP retry handler factory + public AzureTextEmbeddings(string modelId, string endpoint, string apiKey, string apiVersion, ILogger? log = null, + IDelegatingHandlerFactory? handlerFactory = null) + : base(log, handlerFactory) { Verify.NotEmpty(modelId, "The ID cannot be empty, you must provide a Model ID or a Deployment name."); this._modelId = modelId; diff --git a/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextCompletion.cs b/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextCompletion.cs index 00f6553f7ef2..95ec72da38da 100644 --- a/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextCompletion.cs +++ b/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextCompletion.cs @@ -1,11 +1,13 @@ // Copyright (c) Microsoft. All rights reserved. using System.Net.Http.Headers; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel.AI.OpenAI.Clients; using Microsoft.SemanticKernel.AI.OpenAI.HttpSchema; using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Reliability; using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.AI.OpenAI.Services; @@ -27,8 +29,10 @@ public sealed class OpenAITextCompletion : OpenAIClientAbstract, ITextCompletion /// OpenAI API key, see https://platform.openai.com/account/api-keys /// OpenAI organization id. This is usually optional unless your account belongs to multiple organizations. /// Logger - public OpenAITextCompletion(string modelId, string apiKey, string? organization = null, ILogger? log = null) : - base(log) + /// Retry handler + public OpenAITextCompletion(string modelId, string apiKey, string? organization = null, ILogger? log = null, + IDelegatingHandlerFactory? handlerFactory = null) : + base(log, handlerFactory) { Verify.NotEmpty(modelId, "The OpenAI model ID cannot be empty"); this._modelId = modelId; @@ -47,9 +51,10 @@ public OpenAITextCompletion(string modelId, string apiKey, string? organization /// /// The prompt to complete. /// Request settings for the completion API + /// Cancellation token /// The completed text /// AIException thrown during the request - public async Task CompleteAsync(string text, CompleteRequestSettings requestSettings) + public async Task CompleteAsync(string text, CompleteRequestSettings requestSettings, CancellationToken cancellationToken = default) { Verify.NotNull(requestSettings, "Completion settings cannot be empty"); @@ -74,6 +79,6 @@ public async Task CompleteAsync(string text, CompleteRequestSettings req Stop = requestSettings.StopSequences is { Count: > 0 } ? requestSettings.StopSequences : null, }); - return await this.ExecuteCompleteRequestAsync(url, requestBody); + return await this.ExecuteCompleteRequestAsync(url, requestBody, cancellationToken); } } diff --git a/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextEmbeddings.cs b/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextEmbeddings.cs index 6506f4710620..039f0d937c57 100644 --- a/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextEmbeddings.cs +++ b/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextEmbeddings.cs @@ -8,6 +8,7 @@ using Microsoft.SemanticKernel.AI.OpenAI.Clients; using Microsoft.SemanticKernel.AI.OpenAI.HttpSchema; using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Reliability; using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.AI.OpenAI.Services; @@ -30,8 +31,10 @@ public sealed class OpenAITextEmbeddings : OpenAIClientAbstract, IEmbeddingGener /// OpenAI API Key /// Optional OpenAI organization ID, usually required only if your account belongs to multiple organizations /// Application logger - public OpenAITextEmbeddings(string modelId, string apiKey, string? organization = null, ILogger? log = null) - : base(log) + /// Retry handler factory for HTTP requests. + public OpenAITextEmbeddings(string modelId, string apiKey, string? organization = null, ILogger? log = null, + IDelegatingHandlerFactory? handlerFactory = null) + : base(log, handlerFactory) { Verify.NotEmpty(modelId, "The OpenAI model ID cannot be empty"); this._modelId = modelId; diff --git a/dotnet/src/SemanticKernel/Configuration/KernelConfig.cs b/dotnet/src/SemanticKernel/Configuration/KernelConfig.cs index ef8cf8cff877..132b74de66c5 100644 --- a/dotnet/src/SemanticKernel/Configuration/KernelConfig.cs +++ b/dotnet/src/SemanticKernel/Configuration/KernelConfig.cs @@ -3,6 +3,9 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Net; +using System.Net.Http; +using Microsoft.Extensions.Logging.Abstractions; using Microsoft.SemanticKernel.AI.OpenAI.Services; using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Reliability; @@ -15,9 +18,76 @@ namespace Microsoft.SemanticKernel.Configuration; public sealed class KernelConfig { /// - /// Global retry logic used for all the backends + /// Retry configuration for IHttpRetryPolicy that uses RetryAfter header when present. /// - public IRetryMechanism RetryMechanism { get => this._retryMechanism; } + public sealed class HttpRetryConfig + { + /// + /// Maximum number of retries. + /// + /// Thrown when value is negative. + public int MaxRetryCount + { + get { return this._maxRetryCount; } + set + { + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(this.MaxRetryCount), "Max retry count cannot be negative."); + } + + this._maxRetryCount = value; + } + } + + /// + /// Minimum delay between retries. + /// + public TimeSpan MinRetryDelay { get; set; } = TimeSpan.FromSeconds(2); + + /// + /// Maximum delay between retries. + /// + public TimeSpan MaxRetryDelay { get; set; } = TimeSpan.FromSeconds(60); + + /// + /// Maximum total time spent retrying. + /// + public TimeSpan MaxTotalRetryTime { get; set; } = TimeSpan.FromMinutes(2); + + /// + /// Whether to use exponential backoff or not. + /// + public bool UseExponentialBackoff { get; set; } + + /// + /// List of status codes that should be retried. + /// + public List RetryableStatusCodes { get; set; } = new() + { + HttpStatusCode.RequestTimeout, + HttpStatusCode.ServiceUnavailable, + HttpStatusCode.GatewayTimeout, + HttpStatusCode.TooManyRequests + }; + + /// + /// List of exception types that should be retried. + /// + public List RetryableExceptionTypes { get; set; } = new() + { + typeof(HttpRequestException) + }; + + private int _maxRetryCount = 1; + } + + /// + /// Global retry logic used for all the backends http calls + /// + public IDelegatingHandlerFactory HttpHandlerFactory { get; private set; } = new DefaultHttpRetryHandlerFactory(new HttpRetryConfig()); + + public HttpRetryConfig DefaultHttpRetryConfig { get; private set; } = new(); /// /// Adds an Azure OpenAI backend to the list. @@ -180,13 +250,27 @@ public bool HasEmbeddingsBackend(string label, Func? condi } /// - /// Set the retry mechanism to use for the kernel. + /// Set the http retry handler factory to use for the kernel. /// - /// Retry mechanism to use. + /// Http retry handler factory to use. /// The updated kernel configuration. - public KernelConfig SetRetryMechanism(IRetryMechanism? retryMechanism = null) + public KernelConfig SetHttpRetryHandlerFactory(IDelegatingHandlerFactory? httpHandlerFactory = null) + { + if (httpHandlerFactory != null) + { + this.HttpHandlerFactory = httpHandlerFactory; + } + + return this; + } + + public KernelConfig SetDefaultHttpRetryConfig(HttpRetryConfig? httpRetryConfig) { - this._retryMechanism = retryMechanism ?? new PassThroughWithoutRetry(); + if (httpRetryConfig != null) + { + this.DefaultHttpRetryConfig = httpRetryConfig; + } + return this; } @@ -397,7 +481,6 @@ public KernelConfig RemoveAllBackends() private Dictionary EmbeddingsBackends { get; set; } = new(); private string? _defaultCompletionBackend; private string? _defaultEmbeddingsBackend; - private IRetryMechanism _retryMechanism = new PassThroughWithoutRetry(); #endregion } diff --git a/dotnet/src/SemanticKernel/Kernel.cs b/dotnet/src/SemanticKernel/Kernel.cs index 5bf11bbba4b1..a90214b31326 100644 --- a/dotnet/src/SemanticKernel/Kernel.cs +++ b/dotnet/src/SemanticKernel/Kernel.cs @@ -172,10 +172,7 @@ public async Task RunAsync(ContextVariables variables, CancellationTo try { cancellationToken.ThrowIfCancellationRequested(); - await this._config.RetryMechanism.ExecuteWithRetryAsync( - async () => { context = await f.InvokeAsync(context); }, - this._log, - cancellationToken); + context = await f.InvokeAsync(context); if (context.ErrorOccurred) { @@ -268,7 +265,8 @@ private ISKFunction CreateSemanticFunction( azureBackendConfig.Endpoint, azureBackendConfig.APIKey, azureBackendConfig.APIVersion, - this._log)); + this._log, + this._config.HttpHandlerFactory)); break; case OpenAIConfig openAiConfig: @@ -276,7 +274,8 @@ private ISKFunction CreateSemanticFunction( openAiConfig.ModelId, openAiConfig.APIKey, openAiConfig.OrgId, - this._log)); + this._log, + this._config.HttpHandlerFactory)); break; default: diff --git a/dotnet/src/SemanticKernel/KernelBuilder.cs b/dotnet/src/SemanticKernel/KernelBuilder.cs index d039eb10e671..6c5a79fd6c33 100644 --- a/dotnet/src/SemanticKernel/KernelBuilder.cs +++ b/dotnet/src/SemanticKernel/KernelBuilder.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.Net.Http; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.SemanticKernel.AI.Embeddings; @@ -8,6 +9,7 @@ using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.KernelExtensions; using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Reliability; using Microsoft.SemanticKernel.SkillDefinition; using Microsoft.SemanticKernel.TemplateEngine; @@ -23,6 +25,7 @@ public sealed class KernelBuilder private ISemanticTextMemory _memory = NullMemory.Instance; private ILogger _log = NullLogger.Instance; private IMemoryStore? _memoryStorage = null; + private IDelegatingHandlerFactory? _httpHandlerFactory = null; /// /// Create a new kernel instance @@ -40,6 +43,8 @@ public static IKernel Create() /// Kernel instance public IKernel Build() { + this._config.SetHttpRetryHandlerFactory(this._httpHandlerFactory ?? new DefaultHttpRetryHandlerFactory(this._config.DefaultHttpRetryConfig)); + var instance = new Kernel( new SkillCollection(this._log), new PromptTemplateEngine(this._log), @@ -108,6 +113,13 @@ public KernelBuilder WithMemoryStorageAndEmbeddingGenerator( return this; } + public KernelBuilder WithRetryHandlerFactory(IDelegatingHandlerFactory httpHandlerFactory) + { + Verify.NotNull(httpHandlerFactory, "The retry handler factory instance provided is NULL"); + this._httpHandlerFactory = httpHandlerFactory; + return this; + } + /// /// Use the given configuration with the kernel to be built. /// diff --git a/dotnet/src/SemanticKernel/KernelExtensions/MemoryConfiguration.cs b/dotnet/src/SemanticKernel/KernelExtensions/MemoryConfiguration.cs index eb54f57f5056..76430561e22c 100644 --- a/dotnet/src/SemanticKernel/KernelExtensions/MemoryConfiguration.cs +++ b/dotnet/src/SemanticKernel/KernelExtensions/MemoryConfiguration.cs @@ -51,7 +51,8 @@ public static void UseMemory(this IKernel kernel, string? embeddingsBackendLabel azureAIConfig.Endpoint, azureAIConfig.APIKey, azureAIConfig.APIVersion, - kernel.Log); + kernel.Log, + kernel.Config.HttpHandlerFactory); break; case OpenAIConfig openAIConfig: @@ -59,7 +60,8 @@ public static void UseMemory(this IKernel kernel, string? embeddingsBackendLabel openAIConfig.ModelId, openAIConfig.APIKey, openAIConfig.OrgId, - kernel.Log); + kernel.Log, + kernel.Config.HttpHandlerFactory); break; default: diff --git a/dotnet/src/SemanticKernel/Orchestration/SKFunction.cs b/dotnet/src/SemanticKernel/Orchestration/SKFunction.cs index f5b29153419c..b7061f07d9c3 100644 --- a/dotnet/src/SemanticKernel/Orchestration/SKFunction.cs +++ b/dotnet/src/SemanticKernel/Orchestration/SKFunction.cs @@ -109,7 +109,7 @@ async Task LocalFunc( { string prompt = await functionConfig.PromptTemplate.RenderAsync(context); - string completion = await client.CompleteAsync(prompt, requestSettings); + string completion = await client.CompleteAsync(prompt, requestSettings, context.CancellationToken); context.Variables.Update(completion); } #pragma warning disable CA1031 // We need to catch all exceptions to handle the execution state diff --git a/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandler.cs b/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandler.cs new file mode 100644 index 000000000000..c0bf8bedb885 --- /dev/null +++ b/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandler.cs @@ -0,0 +1,256 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.Configuration; + +namespace Microsoft.SemanticKernel.Reliability; + +internal sealed class DefaultHttpRetryHandler : DelegatingHandler +{ + /// + /// Initializes a new instance of the class. + /// + /// The retry configuration. + /// The logger. + public DefaultHttpRetryHandler(KernelConfig.HttpRetryConfig? config = null, ILogger? log = null) : this(config ?? new KernelConfig.HttpRetryConfig(), log, + null, null) + { + } + + public readonly Guid Id = Guid.NewGuid(); + + internal DefaultHttpRetryHandler(KernelConfig.HttpRetryConfig config, ILogger? log = null, IDelayProvider? delayProvider = null, + ITimeProvider? timeProvider = null) + { + this._config = config; + this._log = log ?? NullLogger.Instance; + this._delayProvider = delayProvider ?? new TaskDelayProvider(); + this._timeProvider = timeProvider ?? new DefaultTimeProvider(); + } + + /// + /// Executes the action with retry logic + /// + /// + /// The request is retried if it throws an exception that is a retryable exception. + /// If the request throws an exception that is not a retryable exception, it is not retried. + /// If the request returns a response with a retryable error code, it is retried. + /// If the request returns a response with a non-retryable error code, it is not retried. + /// If the exception contains a RetryAfter header, the request is retried after the specified delay. + /// If configured to use exponential backoff, the delay is doubled for each retry. + /// + /// The request. + /// The cancellation token. + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + int retryCount = 0; + + var start = this._timeProvider.GetCurrentTime(); + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + TimeSpan waitFor = default; + string reason = string.Empty; + HttpResponseMessage? response = null; + try + { + response = await base.SendAsync(request, cancellationToken); + + // If the request does not require a retry then we're done + if (!this.ShouldRetry(response.StatusCode)) + { + return response; + } + + // Drain response content to free connections. Need to perform this + // before retry attempt and before the TooManyRetries ServiceException. + if (response.Content != null) + { +#if NET5_0_OR_GREATER + await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); +#else + await response.Content.ReadAsByteArrayAsync().ConfigureAwait(false); +#endif + } + + reason = response.StatusCode.ToString(); + + if (retryCount >= this._config.MaxRetryCount) + { + this._log.LogError( + "Error executing request, max retry count reached. Reason: {0}", reason); + return response; + } + + // If the retry delay is longer than the total timeout, then we'll + // just return + if (!this.HasTimeForRetry(start, retryCount, response, out waitFor)) + { + this._log.LogError( + "Error executing request, max total retry time reached. Reason: {0}", reason); + return response; + } + } + catch (Exception e) when ((this.ShouldRetry(e) || this.ShouldRetry(e.InnerException)) && + retryCount < this._config.MaxRetryCount && + this.HasTimeForRetry(start, retryCount, response, out waitFor)) + { + reason = e.GetType().ToString(); + } + + // If the request requires a retry then we'll retry + this._log.LogWarning( + "Error executing action [attempt {0} of {1}]. Reason: {2}. Will retry after {3}ms", + retryCount + 1, + this._config.MaxRetryCount, + reason, + waitFor.TotalMilliseconds); + + // Clone request with CloneAsync before retrying + // Do not dispose this request as that breaks the request cloning +#pragma warning disable CA2000 + request = await CloneAsync(request); +#pragma warning restore CA2000 + + // Increase retryCount + retryCount++; + + // Delay + await this._delayProvider.DelayAsync(waitFor, cancellationToken).ConfigureAwait(false); + } + } + + private TimeSpan GetWaitTime(int retryCount, HttpResponseMessage? response) + { + var retryAfter = response?.Headers.RetryAfter?.Date.HasValue == true + ? response?.Headers.RetryAfter?.Date - DateTimeOffset.Now + : (response?.Headers.RetryAfter?.Delta) ?? this._config.MinRetryDelay; + retryAfter ??= this._config.MinRetryDelay; + + var timeToWait = retryAfter > this._config.MaxRetryDelay + ? this._config.MaxRetryDelay + : retryAfter < this._config.MinRetryDelay + ? this._config.MinRetryDelay + : retryAfter ?? default; + + if (this._config.UseExponentialBackoff) + { + for (var backoffRetryCount = 1; backoffRetryCount < retryCount + 1; backoffRetryCount++) + { + timeToWait = timeToWait.Add(timeToWait); + } + } + + return timeToWait; + } + + private bool HasTimeForRetry(DateTimeOffset start, int retryCount, HttpResponseMessage? response, out TimeSpan waitFor) + { + waitFor = this.GetWaitTime(retryCount, response); + var currentTIme = this._timeProvider.GetCurrentTime(); + var result = currentTIme - start + waitFor; + + return result < this._config.MaxTotalRetryTime; + } + + private bool ShouldRetry(HttpStatusCode statusCode) + { + return this._config.RetryableStatusCodes.Contains(statusCode); + } + + private bool ShouldRetry(Exception exception) + { + return this._config.RetryableExceptionTypes.Contains(exception.GetType()); + } + + /// + /// Create a new HTTP request by copying previous HTTP request's headers and properties from response's request message. + /// Copied from: https://github.com/microsoftgraph/msgraph-sdk-dotnet-core/blob/dev/src/Microsoft.Graph.Core/Extensions/HttpRequestMessageExtensions.cs + /// + /// The previous needs to be copy. + /// The . + /// + /// Re-issue a new HTTP request with the previous request's headers and properties + /// + internal static async Task CloneAsync(HttpRequestMessage originalRequest) + { + var newRequest = new HttpRequestMessage(originalRequest.Method, originalRequest.RequestUri); + + // Copy request headers. + foreach (var header in originalRequest.Headers) + { + newRequest.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + + // Copy request properties. + foreach (var property in originalRequest.Properties) + { + newRequest.Properties.Add(property); + } + + // Set Content if previous request had one. + if (originalRequest.Content != null) + { + // HttpClient doesn't rewind streams and we have to explicitly do so. + await originalRequest.Content.ReadAsStreamAsync().ContinueWith(t => + { + if (t.Result.CanSeek) + { + t.Result.Seek(0, SeekOrigin.Begin); + } + + newRequest.Content = new StreamContent(t.Result); + }, TaskScheduler.Current).ConfigureAwait(false); + + // Copy content headers. + if (originalRequest.Content.Headers != null) + { + foreach (var contentHeader in originalRequest.Content.Headers) + { + newRequest.Content.Headers.TryAddWithoutValidation(contentHeader.Key, contentHeader.Value); + } + } + } + + return newRequest; + } + + private readonly KernelConfig.HttpRetryConfig _config; + private readonly ILogger _log; + private readonly IDelayProvider _delayProvider; + private readonly ITimeProvider _timeProvider; + + internal interface IDelayProvider + { + Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken); + } + + internal class TaskDelayProvider : IDelayProvider + { + public async Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken) + { + await Task.Delay(delay, cancellationToken); + } + } + + internal interface ITimeProvider + { + DateTimeOffset GetCurrentTime(); + } + + internal class DefaultTimeProvider : ITimeProvider + { + public DateTimeOffset GetCurrentTime() + { + return DateTimeOffset.UtcNow; + } + } +} diff --git a/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandlerFactory.cs b/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandlerFactory.cs new file mode 100644 index 000000000000..5651bed728cd --- /dev/null +++ b/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandlerFactory.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Configuration; + +namespace Microsoft.SemanticKernel.Reliability; + +internal class DefaultHttpRetryHandlerFactory : IDelegatingHandlerFactory +{ + internal DefaultHttpRetryHandlerFactory(KernelConfig.HttpRetryConfig? config = null) + { + this._config = config; + } + + public DelegatingHandler Create(ILogger log) + { + return new DefaultHttpRetryHandler(this._config, log); + } + + private readonly KernelConfig.HttpRetryConfig? _config; +} diff --git a/dotnet/src/SemanticKernel/Reliability/IDelegatingHandlerFactory.cs b/dotnet/src/SemanticKernel/Reliability/IDelegatingHandlerFactory.cs new file mode 100644 index 000000000000..1d4d4d29f8ae --- /dev/null +++ b/dotnet/src/SemanticKernel/Reliability/IDelegatingHandlerFactory.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Microsoft.Extensions.Logging; + +namespace Microsoft.SemanticKernel.Reliability; + +/// +/// Factory for creating instances. +/// +public interface IDelegatingHandlerFactory +{ + DelegatingHandler Create(ILogger log); +} diff --git a/dotnet/src/SemanticKernel/Reliability/IRetryMechanism.cs b/dotnet/src/SemanticKernel/Reliability/IRetryMechanism.cs deleted file mode 100644 index cd92fc8e278c..000000000000 --- a/dotnet/src/SemanticKernel/Reliability/IRetryMechanism.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; - -namespace Microsoft.SemanticKernel.Reliability; - -/// -/// Interface for retry mechanisms on AI calls. -/// -public interface IRetryMechanism -{ - /// - /// Executes the given action with retry logic. - /// - /// The action to retry on exception. - /// The logger to use. - /// The cancellation token. - /// An awaitable task. - Task ExecuteWithRetryAsync(Func action, ILogger log, CancellationToken cancellationToken = default); -} diff --git a/dotnet/src/SemanticKernel/Reliability/NullHttpRetryHandler.cs b/dotnet/src/SemanticKernel/Reliability/NullHttpRetryHandler.cs new file mode 100644 index 000000000000..8749ace9ef51 --- /dev/null +++ b/dotnet/src/SemanticKernel/Reliability/NullHttpRetryHandler.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Microsoft.Extensions.Logging; + +namespace Microsoft.SemanticKernel.Reliability; + +public class NullHttpRetryHandlerFactory : IDelegatingHandlerFactory +{ + public DelegatingHandler Create(ILogger log) + { + return new NullHttpRetryHandler(); + } +} + +/// +/// A http retry handler that does not retry. +/// +public class NullHttpRetryHandler : DelegatingHandler +{ +} diff --git a/dotnet/src/SemanticKernel/Reliability/PassThroughWithoutRetry.cs b/dotnet/src/SemanticKernel/Reliability/PassThroughWithoutRetry.cs deleted file mode 100644 index 8680722dae04..000000000000 --- a/dotnet/src/SemanticKernel/Reliability/PassThroughWithoutRetry.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.Diagnostics; - -namespace Microsoft.SemanticKernel.Reliability; - -/// -/// A retry mechanism that does not retry. -/// -internal class PassThroughWithoutRetry : IRetryMechanism -{ - public async Task ExecuteWithRetryAsync(Func action, ILogger log, CancellationToken cancellationToken = default) - { - try - { - if (!cancellationToken.IsCancellationRequested) - { - await action(); - } - } - catch (Exception ex) when (!ex.IsCriticalException()) - { - log.LogWarning(ex, "Error executing action, not retrying"); - throw; - } - } -} diff --git a/dotnet/src/SemanticKernel/SemanticKernel.csproj b/dotnet/src/SemanticKernel/SemanticKernel.csproj index e22aa0fba3ce..fb9b247d3912 100644 --- a/dotnet/src/SemanticKernel/SemanticKernel.csproj +++ b/dotnet/src/SemanticKernel/SemanticKernel.csproj @@ -39,5 +39,8 @@ <_Parameter1>SemanticKernelTests + + <_Parameter1>DynamicProxyGenAssembly2 + \ No newline at end of file diff --git a/samples/dotnet/KernelBuilder/Program.cs b/samples/dotnet/KernelBuilder/Program.cs index c542a2dfe811..2fef62390d24 100644 --- a/samples/dotnet/KernelBuilder/Program.cs +++ b/samples/dotnet/KernelBuilder/Program.cs @@ -104,15 +104,33 @@ // AI requests (when using the kernel). var kernel8 = Kernel.Builder - .Configure(c => c.SetRetryMechanism(new RetryThreeTimes())) + .Configure(c => c.SetHttpRetryHandlerFactory(new RetryThreeTimesFactory())) .Build(); -public class RetryThreeTimes : IRetryMechanism +public class RetryThreeTimesFactory : IDelegatingHandlerFactory { - public Task ExecuteWithRetryAsync(Func action, ILogger log, CancellationToken cancellationToken = default) + public DelegatingHandler Create(ILogger log) { - var policy = GetPolicy(log); - return policy.ExecuteAsync((_) => action(), cancellationToken); + return new RetryThreeTimes(log); + } +} + +public class RetryThreeTimes : DelegatingHandler +{ + private readonly AsyncRetryPolicy _policy; + + public RetryThreeTimes(ILogger log = null) + { + this._policy = GetPolicy(log ?? NullLogger.Instance); + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + return await this._policy.ExecuteAsync(async () => + { + var response = await base.SendAsync(request, cancellationToken); + return response; + }); } private static AsyncRetryPolicy GetPolicy(ILogger log) @@ -126,7 +144,7 @@ private static AsyncRetryPolicy GetPolicy(ILogger log) TimeSpan.FromSeconds(8) }, (ex, timespan, retryCount, _) => log.LogWarning(ex, - "Error executing action [attempt {0} of ], pausing {1} msecs", + "Error executing action [attempt {0} of 3], pausing {1}ms", retryCount, timespan.TotalMilliseconds)); } } diff --git a/samples/dotnet/kernel-syntax-examples/Example08_RetryHandler.cs b/samples/dotnet/kernel-syntax-examples/Example08_RetryHandler.cs new file mode 100644 index 000000000000..812531bd80f2 --- /dev/null +++ b/samples/dotnet/kernel-syntax-examples/Example08_RetryHandler.cs @@ -0,0 +1,175 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Configuration; +using Microsoft.SemanticKernel.CoreSkills; +using Microsoft.SemanticKernel.KernelExtensions; +using Microsoft.SemanticKernel.Reliability; +using Reliability; +using RepoUtils; + +// ReSharper disable once InconsistentNaming +public static class Example08_RetryHandler +{ + public static async Task RunAsync() + { + var kernel = InitializeKernel(); + var retryHandlerFactory = new RetryThreeTimesWithBackoffFactory(); + ConsoleLogger.Log.LogInformation("============================== RetryThreeTimesWithBackoff =============================="); + await RunRetryPolicyAsync(kernel, retryHandlerFactory); + + ConsoleLogger.Log.LogInformation("========================= RetryThreeTimesWithRetryAfterBackoff ========================="); + await RunRetryPolicyBuilderAsync(typeof(RetryThreeTimesWithRetryAfterBackoffFactory)); + + ConsoleLogger.Log.LogInformation("==================================== NoRetryPolicy ====================================="); + await RunRetryPolicyBuilderAsync(typeof(NullHttpRetryHandlerFactory)); + + ConsoleLogger.Log.LogInformation("=============================== DefaultHttpRetryHandler ================================"); + await RunRetryHandlerConfigAsync(new KernelConfig.HttpRetryConfig() { MaxRetryCount = 3, UseExponentialBackoff = true }); + + ConsoleLogger.Log.LogInformation("======= DefaultHttpRetryConfig [MaxRetryCount = 3, UseExponentialBackoff = true] ======="); + await RunRetryHandlerConfigAsync(new KernelConfig.HttpRetryConfig() { MaxRetryCount = 3, UseExponentialBackoff = true }); + } + + private static async Task RunRetryHandlerConfigAsync(KernelConfig.HttpRetryConfig? config = null) + { + var kernelBuilder = Kernel.Builder.WithLogger(ConsoleLogger.Log); + if (config != null) + { + kernelBuilder = kernelBuilder.Configure(c => c.SetDefaultHttpRetryConfig(config)); + } + + // Add 401 to the list of retryable status codes + // Typically 401 would not be something we retry but for demonstration + // purposes we are doing so as it's easy to trigger when using an invalid key. + kernelBuilder = kernelBuilder.Configure(c => c.DefaultHttpRetryConfig.RetryableStatusCodes.Add(System.Net.HttpStatusCode.Unauthorized)); + + // OpenAI settings - you can set the OPENAI_API_KEY to an invalid value to see the retry policy in play + kernelBuilder = kernelBuilder.Configure(c => c.AddOpenAICompletionBackend("text-davinci-003", "text-davinci-003", "BAD_KEY")); + + var kernel = kernelBuilder.Build(); + + await ImportAndExecuteSkillAsync(kernel); + } + + private static IKernel InitializeKernel() + { + var kernel = Kernel.Builder.WithLogger(ConsoleLogger.Log).Build(); + // OpenAI settings - you can set the OPENAI_API_KEY to an invalid value to see the retry policy in play + kernel.Config.AddOpenAICompletionBackend("text-davinci-003", "text-davinci-003", "BAD_KEY"); + + return kernel; + } + + private static async Task RunRetryPolicyAsync(IKernel kernel, IDelegatingHandlerFactory retryHandlerFactory) + { + kernel.Config.SetHttpHandlerFactory(retryHandlerFactory); + await ImportAndExecuteSkillAsync(kernel); + } + + private static async Task RunRetryPolicyBuilderAsync(Type retryHandlerFactoryType) + { + var kernelBuilder = Kernel.Builder.WithLogger(ConsoleLogger.Log) + .WithRetryHandlerFactory((Activator.CreateInstance(retryHandlerFactoryType) as IDelegatingHandlerFactory)!); + + // OpenAI settings - you can set the OPENAI_API_KEY to an invalid value to see the retry policy in play + kernelBuilder = kernelBuilder.Configure(c => c.AddOpenAICompletionBackend("text-davinci-003", "text-davinci-003", "BAD_KEY")); + + var kernel = kernelBuilder.Build(); + + await ImportAndExecuteSkillAsync(kernel); + } + + private static async Task ImportAndExecuteSkillAsync(IKernel kernel) + { + // Load semantic skill defined with prompt templates + string folder = RepoFiles.SampleSkillsPath(); + + kernel.ImportSkill(new TimeSkill(), "time"); + + var qaSkill = kernel.ImportSemanticSkillFromDirectory( + folder, + "QASkill"); + + var question = "How popular is Polly library?"; + + ConsoleLogger.Log.LogInformation("Question: {0}", question); + // To see the retry policy in play, you can set the OPENAI_API_KEY to an invalid value + var answer = await kernel.RunAsync(question, qaSkill["Question"]); + ConsoleLogger.Log.LogInformation("Answer: {0}", answer); + } +} + +/* Output: +info: object[0] + ============================== RetryThreeTimesWithBackoff ============================== +info: object[0] + Question: How popular is Polly library? +warn: object[0] + Error executing action [attempt 1 of 3], pausing 2000ms. Outcome: Unauthorized +warn: object[0] + Error executing action [attempt 2 of 3], pausing 4000ms. Outcome: Unauthorized +warn: object[0] + Error executing action [attempt 3 of 3], pausing 8000ms. Outcome: Unauthorized +fail: object[0] + Function call fail during pipeline step 0: QASkill.Question +info: object[0] + Answer: Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized +info: object[0] + ========================= RetryThreeTimesWithRetryAfterBackoff ========================= +info: object[0] + Question: How popular is Polly library? +warn: object[0] + Error executing action [attempt 1 of 3], pausing 2000ms. Outcome: Unauthorized +warn: object[0] + Error executing action [attempt 2 of 3], pausing 2000ms. Outcome: Unauthorized +warn: object[0] + Error executing action [attempt 3 of 3], pausing 2000ms. Outcome: Unauthorized +fail: object[0] + Function call fail during pipeline step 0: QASkill.Question +info: object[0] + Answer: Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized +info: object[0] + ==================================== NoRetryPolicy ===================================== +info: object[0] + Question: How popular is Polly library? +fail: object[0] + Function call fail during pipeline step 0: QASkill.Question +info: object[0] + Answer: Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized +info: object[0] + =============================== DefaultHttpRetryHandler ================================ +info: object[0] + Question: How popular is Polly library? +warn: object[0] + Error executing action [attempt 1 of 3]. Reason: Unauthorized. Will retry after 2000ms +warn: object[0] + Error executing action [attempt 2 of 3]. Reason: Unauthorized. Will retry after 4000ms +warn: object[0] + Error executing action [attempt 3 of 3]. Reason: Unauthorized. Will retry after 8000ms +fail: object[0] + Error executing request, max retry count reached. Reason: Unauthorized +fail: object[0] + Function call fail during pipeline step 0: QASkill.Question +info: object[0] + Answer: Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized +info: object[0] + ======= DefaultHttpRetryConfig [MaxRetryCount = 3, UseExponentialBackoff = true] ======= +info: object[0] + Question: How popular is Polly library? +warn: object[0] + Error executing action [attempt 1 of 3]. Reason: Unauthorized. Will retry after 2000ms +warn: object[0] + Error executing action [attempt 2 of 3]. Reason: Unauthorized. Will retry after 4000ms +warn: object[0] + Error executing action [attempt 3 of 3]. Reason: Unauthorized. Will retry after 8000ms +fail: object[0] + Error executing request, max retry count reached. Reason: Unauthorized +fail: object[0] + Function call fail during pipeline step 0: QASkill.Question +info: object[0] + Answer: Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized +*/ diff --git a/samples/dotnet/kernel-syntax-examples/Example08_RetryMechanism.cs b/samples/dotnet/kernel-syntax-examples/Example08_RetryMechanism.cs deleted file mode 100644 index 115d87b62e6a..000000000000 --- a/samples/dotnet/kernel-syntax-examples/Example08_RetryMechanism.cs +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.CoreSkills; -using Microsoft.SemanticKernel.KernelExtensions; -using Reliability; -using RepoUtils; - -// ReSharper disable once InconsistentNaming -public static class Example08_RetryMechanism -{ - public static async Task RunAsync() - { - Console.WriteLine("============================ RetryMechanism ============================"); - - IKernel kernel = Kernel.Builder.WithLogger(ConsoleLogger.Log).Build(); - kernel.Config.SetRetryMechanism(new RetryThreeTimesWithBackoff()); - - // OpenAI settings - kernel.Config.AddOpenAICompletionBackend("text-davinci-003", "text-davinci-003", Env.Var("OPENAI_API_KEY")); - - // Load semantic skill defined with prompt templates - string folder = RepoFiles.SampleSkillsPath(); - - kernel.ImportSkill(new TimeSkill(), "time"); - - var qaSkill = kernel.ImportSemanticSkillFromDirectory( - folder, - "QASkill"); - - var question = "How popular is Polly library?"; - var answer = await kernel.RunAsync(question, qaSkill["Question"]); - - Console.WriteLine($"Question: {question}\n\n" + answer); - } -} diff --git a/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs b/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs index f0038055c222..6bc930db92d3 100644 --- a/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs +++ b/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs @@ -9,7 +9,6 @@ using Microsoft.SemanticKernel.KernelExtensions; using Microsoft.SemanticKernel.Orchestration; using Microsoft.SemanticKernel.Orchestration.Extensions; -using Reliability; using RepoUtils; using Skills; @@ -46,7 +45,7 @@ private static async Task PoetrySamplesAsync() Console.WriteLine("Original plan:"); Console.WriteLine(originalPlan.Variables.ToPlan().PlanString); - _ = await ExecutePlanAsync(kernel, planner, originalPlan, 5); + await ExecutePlanAsync(kernel, planner, originalPlan, 5); } private static async Task EmailSamplesAsync() @@ -87,7 +86,7 @@ private static async Task EmailSamplesAsync() "and the kingdom was at peace once again. The king was so grateful to Mira that he asked her to marry him and she agreed. " + "They ruled the kingdom together, ruling with fairness and compassion, just as Arjun had done before. They lived " + "happily ever after, with the people of the kingdom remembering Mira as the brave young woman who saved them from the dragon."); - _ = await ExecutePlanAsync(kernel, planner, executionResults, 5); + await ExecutePlanAsync(kernel, planner, executionResults, 5); } private static async Task BookSamplesAsync() @@ -118,7 +117,7 @@ private static async Task BookSamplesAsync() Stopwatch sw = new(); sw.Start(); - _ = await ExecutePlanAsync(kernel, planner, originalPlan); + await ExecutePlanAsync(kernel, planner, originalPlan); } private static IKernel InitializeKernelAndPlanner(out IDictionary planner) @@ -129,7 +128,6 @@ private static IKernel InitializeKernelAndPlanner(out IDictionary + diff --git a/samples/dotnet/kernel-syntax-examples/Program.cs b/samples/dotnet/kernel-syntax-examples/Program.cs index c6ccf7aaf2ab..ef946a5c4fb4 100644 --- a/samples/dotnet/kernel-syntax-examples/Program.cs +++ b/samples/dotnet/kernel-syntax-examples/Program.cs @@ -30,7 +30,7 @@ public static async Task Main() await Example07_TemplateLanguage.RunAsync(); Console.WriteLine("== DONE =="); - await Example08_RetryMechanism.RunAsync(); + await Example08_RetryHandler.RunAsync(); Console.WriteLine("== DONE =="); await Example09_FunctionTypes.RunAsync(); diff --git a/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithBackoff.cs b/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithBackoff.cs index 157fe10c5acb..bc7e644b696a 100644 --- a/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithBackoff.cs +++ b/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithBackoff.cs @@ -1,38 +1,63 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.Reliability; using Polly; using Polly.Retry; namespace Reliability; +public class RetryThreeTimesWithBackoffFactory : IDelegatingHandlerFactory +{ + public DelegatingHandler Create(ILogger log) + { + return new RetryThreeTimesWithBackoff(log); + } +} + /// /// An example of a retry mechanism that retries three times with backoff. /// -public class RetryThreeTimesWithBackoff : IRetryMechanism +public class RetryThreeTimesWithBackoff : DelegatingHandler { - public Task ExecuteWithRetryAsync(Func action, ILogger log, CancellationToken cancellationToken = default) + private readonly AsyncRetryPolicy _policy; + + public RetryThreeTimesWithBackoff(ILogger log) + { + this._policy = GetPolicy(log); + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { - var policy = GetPolicy(log); - return policy.ExecuteAsync((_) => action(), cancellationToken); + return await this._policy.ExecuteAsync(async () => + { + var response = await base.SendAsync(request, cancellationToken); + return response; + }); } - private static AsyncRetryPolicy GetPolicy(ILogger log) + private static AsyncRetryPolicy GetPolicy(ILogger log) { + // Handle 429 and 401 errors + // Typically 401 would not be something we retry but for demonstration + // purposes we are doing so as it's easy to trigger when using an invalid key. return Policy - .Handle(ex => ex.ErrorCode == AIException.ErrorCodes.Throttling) + .HandleResult(response => + response.StatusCode is System.Net.HttpStatusCode.TooManyRequests or System.Net.HttpStatusCode.Unauthorized) .WaitAndRetryAsync(new[] { TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(4), TimeSpan.FromSeconds(8) }, - (ex, timespan, retryCount, _) => log.LogWarning(ex, - "Error executing action [attempt {0} of ], pausing {1} msecs", retryCount, timespan.TotalMilliseconds)); + (outcome, timespan, retryCount, _) => log.LogWarning( + "Error executing action [attempt {0} of 3], pausing {1}ms. Outcome: {2}", + retryCount, + timespan.TotalMilliseconds, + outcome.Result.StatusCode)); } } diff --git a/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithRetryAfterBackoff.cs b/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithRetryAfterBackoff.cs new file mode 100644 index 000000000000..2da172becf75 --- /dev/null +++ b/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithRetryAfterBackoff.cs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Reliability; +using Polly; +using Polly.Retry; + +namespace Reliability; + +public class RetryThreeTimesWithRetryAfterBackoffFactory : IDelegatingHandlerFactory +{ + public DelegatingHandler Create(ILogger log) + { + return new RetryThreeTimesWithRetryAfterBackoff(log); + } +} + +/// +/// An example of a retry mechanism that retries three times with backoff using the RetryAfter value. +/// +public class RetryThreeTimesWithRetryAfterBackoff : DelegatingHandler +{ + private readonly AsyncRetryPolicy _policy; + + public RetryThreeTimesWithRetryAfterBackoff(ILogger log) + { + this._policy = GetPolicy(log); + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + return await this._policy.ExecuteAsync(async () => + { + var response = await base.SendAsync(request, cancellationToken); + return response; + }); + } + + private static AsyncRetryPolicy GetPolicy(ILogger log) + { + // Handle 429 and 401 errors + // Typically 401 would not be something we retry but for demonstration + // purposes we are doing so as it's easy to trigger when using an invalid key. + return Policy + .HandleResult(response => + response.StatusCode is System.Net.HttpStatusCode.TooManyRequests or System.Net.HttpStatusCode.Unauthorized) + .WaitAndRetryAsync( + retryCount: 3, + sleepDurationProvider: (_, r, _) => + { + var response = r.Result; + var retryAfter = response.Headers.RetryAfter?.Delta ?? response.Headers.RetryAfter?.Date - DateTimeOffset.Now; + return retryAfter ?? TimeSpan.FromSeconds(2); + }, + (outcome, timespan, retryCount, _) => + { + log.LogWarning( + "Error executing action [attempt {0} of 3], pausing {1}ms. Outcome: {2}", + retryCount, + timespan.TotalMilliseconds, + outcome.Result.StatusCode); + return Task.CompletedTask; + }); + } +} diff --git a/samples/dotnet/kernel-syntax-examples/RepoUtils/ConsoleLogger.cs b/samples/dotnet/kernel-syntax-examples/RepoUtils/ConsoleLogger.cs index 51aa7055668b..a3d7ff63a8f9 100644 --- a/samples/dotnet/kernel-syntax-examples/RepoUtils/ConsoleLogger.cs +++ b/samples/dotnet/kernel-syntax-examples/RepoUtils/ConsoleLogger.cs @@ -13,18 +13,20 @@ internal static class ConsoleLogger internal static ILogger Log => LogFactory.CreateLogger(); private static ILoggerFactory LogFactory => s_loggerFactory.Value; + private static readonly Lazy s_loggerFactory = new(LogBuilder); private static ILoggerFactory LogBuilder() { return LoggerFactory.Create(builder => { - builder.SetMinimumLevel(LogLevel.Warning); + // builder.SetMinimumLevel(LogLevel.Warning); // builder.AddFilter("Microsoft", LogLevel.Trace); // builder.AddFilter("Microsoft", LogLevel.Debug); // builder.AddFilter("Microsoft", LogLevel.Information); - // builder.AddFilter("Microsoft", LogLevel.Warning); + builder.AddFilter("Microsoft", LogLevel.Warning); // builder.AddFilter("Microsoft", LogLevel.Error); + builder.AddFilter("System", LogLevel.Warning); builder.AddConsole(); }); }