From b53c60d6e0ff66e1a2daeb315028a0cd9346bfd4 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Wed, 8 Mar 2023 23:21:54 -0800 Subject: [PATCH 1/8] This commit adds and refactors various features related to the HTTP retry policy used by the semantic kernel and its extensions. The main changes are: - Replacing the IRetryMechanism interface with the IDelegatingHandlerFactory interface, which allows more flexibility and control over the retry logic for HTTP requests. - Adding a new DefaultHttpRetryHandler class that inherits from DelegatingHandler and implements the default retry policy using Polly. The policy handles retryable status codes and exceptions, respects the RetryAfter header from the server, and uses exponential backoff if configured. - Adding a new NullHttpRetryHandler class that implements a no-retry policy. - Adding a new HttpRetryConfig class that defines the default retry configuration for the IHttpRetryPolicy. - Adding a new HttpHandlerFactory property to the KernelConfig class, which allows users to specify a custom retry handler factory for the kernel. The class also exposes a SetHttpHandlerFactory and a SetDefaultHttpRetryConfig method to configure these properties. - Updating the OpenAI services to use the HttpHandlerFactory from the KernelConfig, and adding a new parameter to their constructors. - Adding unit tests for the DefaultHttpRetryHandler and NullHttpRetryHandler classes, covering various scenarios of retrying and not retrying HTTP requests. - Adding a new file Example08_RetryHandler.cs that demonstrates how to use different retry policies when making HTTP requests with the semantic kernel. - Removing the PassThroughWithoutRetry class and the Example08_RetryMechanism.cs file, as they are no longer needed. - Properly disposing of the retry handler in the OpenAIClientAbstract class. - Adding a CancellationToken parameter to the CompleteAsync method of the ITextCompletion interface and its implementations, allowing the caller to cancel the request if needed. - Refactoring the OpenAIClientAbstract class to use a delegating handler factory for injecting retry logic, and updating the AzureOpenAIClientAbstract class accordingly. - Modifying some documentation and tests to reflect the changes. - Updating the project file to allow Moq to access internal types for testing purposes. - Fixing some typos and formatting issues in the code. --- dotnet/nuget/nuget-package.props | 2 +- .../AI/OpenAICompletionTests.cs | 30 + dotnet/src/IntegrationTest/RedirectOutput.cs | 29 +- .../Configuration/KernelConfigTests.cs | 106 ++- dotnet/src/SemanticKernel.Test/KernelTests.cs | 3 +- .../DefaultHttpRetryHandlerTests.cs | 642 ++++++++++++++++++ .../Reliability/NullHttpRetryHandlerTests.cs | 107 +++ .../PassThroughWithoutRetryTests.cs | 75 -- .../AI/ITextCompletionClient.cs | 4 +- .../Clients/AzureOpenAIClientAbstract.cs | 5 +- .../AI/OpenAI/Clients/OpenAIClientAbstract.cs | 26 +- .../AI/OpenAI/Services/AzureTextCompletion.cs | 13 +- .../AI/OpenAI/Services/AzureTextEmbeddings.cs | 7 +- .../OpenAI/Services/OpenAITextCompletion.cs | 13 +- .../OpenAI/Services/OpenAITextEmbeddings.cs | 7 +- .../Configuration/KernelConfig.cs | 99 ++- dotnet/src/SemanticKernel/Kernel.cs | 11 +- dotnet/src/SemanticKernel/KernelBuilder.cs | 12 + .../KernelExtensions/MemoryConfiguration.cs | 6 +- .../Orchestration/SKFunction.cs | 2 +- .../Reliability/DefaultHttpRetryHandler.cs | 256 +++++++ .../DefaultHttpRetryHandlerFactory.cs | 24 + .../Reliability/IDelegatingHandlerFactory.cs | 13 + .../Reliability/IRetryMechanism.cs | 23 - .../Reliability/NullHttpRetryHandler.cs | 20 + .../Reliability/PassThroughWithoutRetry.cs | 31 - .../src/SemanticKernel/SemanticKernel.csproj | 3 + samples/dotnet/KernelBuilder/Program.cs | 30 +- .../Example08_RetryHandler.cs | 149 ++++ .../Example08_RetryMechanism.cs | 38 -- .../Example12_Planning.cs | 8 +- .../Example13_ConversationSummarySkill.cs | 4 +- .../KernelSyntaxExamples.csproj | 1 + .../dotnet/kernel-syntax-examples/Program.cs | 2 +- .../Reliability/RetryThreeTimesWithBackoff.cs | 50 +- .../RetryThreeTimesWithRetryAfterBackoff.cs | 76 +++ 36 files changed, 1682 insertions(+), 245 deletions(-) create mode 100644 dotnet/src/SemanticKernel.Test/Reliability/DefaultHttpRetryHandlerTests.cs create mode 100644 dotnet/src/SemanticKernel.Test/Reliability/NullHttpRetryHandlerTests.cs delete mode 100644 dotnet/src/SemanticKernel.Test/Reliability/PassThroughWithoutRetryTests.cs create mode 100644 dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandler.cs create mode 100644 dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandlerFactory.cs create mode 100644 dotnet/src/SemanticKernel/Reliability/IDelegatingHandlerFactory.cs delete mode 100644 dotnet/src/SemanticKernel/Reliability/IRetryMechanism.cs create mode 100644 dotnet/src/SemanticKernel/Reliability/NullHttpRetryHandler.cs delete mode 100644 dotnet/src/SemanticKernel/Reliability/PassThroughWithoutRetry.cs create mode 100644 samples/dotnet/kernel-syntax-examples/Example08_RetryHandler.cs delete mode 100644 samples/dotnet/kernel-syntax-examples/Example08_RetryMechanism.cs create mode 100644 samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithRetryAfterBackoff.cs diff --git a/dotnet/nuget/nuget-package.props b/dotnet/nuget/nuget-package.props index 9f95907c4486..24e4f41f93fe 100644 --- a/dotnet/nuget/nuget-package.props +++ b/dotnet/nuget/nuget-package.props @@ -3,7 +3,7 @@ true true - 0.8 + 0.9 diff --git a/dotnet/src/IntegrationTest/AI/OpenAICompletionTests.cs b/dotnet/src/IntegrationTest/AI/OpenAICompletionTests.cs index b7cdadcf4c32..ec882f8a7cf9 100644 --- a/dotnet/src/IntegrationTest/AI/OpenAICompletionTests.cs +++ b/dotnet/src/IntegrationTest/AI/OpenAICompletionTests.cs @@ -8,6 +8,7 @@ using IntegrationTests.TestSettings; using Microsoft.Extensions.Configuration; using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Configuration; using Microsoft.SemanticKernel.KernelExtensions; using Microsoft.SemanticKernel.Orchestration; using Xunit; @@ -91,6 +92,35 @@ public async Task AzureOpenAITestAsync(string prompt, string expectedAnswerConta Assert.Contains(expectedAnswerContains, actual.Result, StringComparison.InvariantCultureIgnoreCase); } + [Theory] + [InlineData("Where is the most famous fish market in Seattle, Washington, USA?", + "Error executing action [attempt 1 of 1]. Reason: Unauthorized. Will retry after 2000ms")] + public async Task OpenAIHttpRetryPolicyTestAsync(string prompt, string expectedOutput) + { + // Arrange + var retryConfig = new KernelConfig.HttpRetryConfig(); + retryConfig.RetryableStatusCodes.Add(System.Net.HttpStatusCode.Unauthorized); + IKernel target = Kernel.Builder.WithLogger(this._testOutputHelper).Configure(c => c.SetDefaultHttpRetryConfig(retryConfig)).Build(); + + OpenAIConfiguration? openAIConfiguration = this._configuration.GetSection("OpenAI").Get(); + Assert.NotNull(openAIConfiguration); + + target.Config.AddOpenAICompletionBackend( + label: openAIConfiguration.Label, + modelId: openAIConfiguration.ModelId, + apiKey: "INVALID_KEY"); + + target.Config.SetDefaultCompletionBackend(openAIConfiguration.Label); + + IDictionary skill = GetSkill("SummarizeSkill", target); + + // Act + await target.RunAsync(prompt, skill["Summarize"]); + + // Assert + Assert.Contains(expectedOutput, this._testOutputHelper.GetLogs(), StringComparison.InvariantCultureIgnoreCase); + } + private static IDictionary GetSkill(string skillName, IKernel target) { string? currentAssemblyDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); diff --git a/dotnet/src/IntegrationTest/RedirectOutput.cs b/dotnet/src/IntegrationTest/RedirectOutput.cs index f3fadf13e8d2..173a1d36136d 100644 --- a/dotnet/src/IntegrationTest/RedirectOutput.cs +++ b/dotnet/src/IntegrationTest/RedirectOutput.cs @@ -1,18 +1,22 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.IO; using System.Text; +using Microsoft.Extensions.Logging; using Xunit.Abstractions; namespace IntegrationTests; -public class RedirectOutput : TextWriter +public class RedirectOutput : TextWriter, ILogger { private readonly ITestOutputHelper _output; + private readonly StringBuilder _logs; public RedirectOutput(ITestOutputHelper output) { this._output = output; + this._logs = new StringBuilder(); } public override Encoding Encoding { get; } = Encoding.UTF8; @@ -20,5 +24,28 @@ public RedirectOutput(ITestOutputHelper output) public override void WriteLine(string? value) { this._output.WriteLine(value); + this._logs.AppendLine(value); + } + + public IDisposable? BeginScope(TState state) where TState : notnull + { + return null; + } + + public bool IsEnabled(LogLevel logLevel) + { + return true; + } + + public string GetLogs() + { + return this._logs.ToString(); + } + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + var message = formatter(state, exception); + this._output?.WriteLine(message); + this._logs.AppendLine(message); } } diff --git a/dotnet/src/SemanticKernel.Test/Configuration/KernelConfigTests.cs b/dotnet/src/SemanticKernel.Test/Configuration/KernelConfigTests.cs index 3dc94106a6fb..e3a620e48314 100644 --- a/dotnet/src/SemanticKernel.Test/Configuration/KernelConfigTests.cs +++ b/dotnet/src/SemanticKernel.Test/Configuration/KernelConfigTests.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using System.Linq; +using System.Threading.Tasks; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.AI.OpenAI.Services; using Microsoft.SemanticKernel.Configuration; @@ -16,55 +17,134 @@ namespace SemanticKernelTests.Configuration; public class KernelConfigTests { [Fact] - public void RetryMechanismIsSet() + public void HttpRetryPolicyIsSet() { // Arrange - var retry = new PassThroughWithoutRetry(); + var retry = new NullHttpRetryHandlerFactory(); var config = new KernelConfig(); // Act - config.SetRetryMechanism(retry); + config.SetHttpHandlerFactory(retry); // Assert - Assert.Equal(retry, config.RetryMechanism); + Assert.Equal(retry, config.HttpHandlerFactory); } [Fact] - public void RetryMechanismIsSetWithCustomImplementation() + public void HttpRetryPolicyIsSetWithCustomImplementation() { // Arrange - var retry = new Mock(); + var retry = new Mock(); var config = new KernelConfig(); // Act - config.SetRetryMechanism(retry.Object); + config.SetHttpHandlerFactory(retry.Object); // Assert - Assert.Equal(retry.Object, config.RetryMechanism); + Assert.Equal(retry.Object, config.HttpHandlerFactory); } [Fact] - public void RetryMechanismIsSetToPassThroughWithoutRetryIfNull() + public void HttpRetryPolicyIsSetToDefaultHttpHandlerFactoryIfNull() { // Arrange var config = new KernelConfig(); // Act - config.SetRetryMechanism(null); + config.SetHttpHandlerFactory(null); // Assert - Assert.IsType(config.RetryMechanism); + Assert.IsType(config.HttpHandlerFactory); } [Fact] - public void RetryMechanismIsSetToPassThroughWithoutRetryIfNotSet() + public void HttpRetryPolicyIsSetToDefaultHttpHandlerFactoryIfNotSet() { // Arrange var config = new KernelConfig(); // Act // Assert - Assert.IsType(config.RetryMechanism); + Assert.IsType(config.HttpHandlerFactory); + } + + [Fact] + public async Task NegativeMaxRetryCountThrowsAsync() + { + // Act + await Assert.ThrowsAsync(() => + { + var httpRetryConfig = new KernelConfig.HttpRetryConfig() { MaxRetryCount = -1 }; + return Task.CompletedTask; + }); + } + + [Fact] + public void SetDefaultHttpRetryConfig() + { + // Arrange + var config = new KernelConfig(); + var httpRetryConfig = new KernelConfig.HttpRetryConfig() { MaxRetryCount = 1 }; + + // Act + config.SetDefaultHttpRetryConfig(httpRetryConfig); + + // Assert + Assert.Equal(httpRetryConfig, config.DefaultHttpRetryConfig); + } + + [Fact] + public void SetDefaultHttpRetryConfigToNull() + { + // Arrange + var config = new KernelConfig(); + var httpRetryConfig = new KernelConfig.HttpRetryConfig() { MaxRetryCount = 1 }; + config.SetDefaultHttpRetryConfig(httpRetryConfig); + + // Act + config.SetDefaultHttpRetryConfig(null); + + // Assert + Assert.Equal(httpRetryConfig, config.DefaultHttpRetryConfig); + } + + // } + [Fact] + public void SetDefaultHttpRetryConfigToDefaultIfNotSet() + { + // Arrange + var config = new KernelConfig(); + + // Act + // Assert + var defaultConfig = new KernelConfig.HttpRetryConfig(); + Assert.Equal(defaultConfig.MaxRetryCount, config.DefaultHttpRetryConfig.MaxRetryCount); + Assert.Equal(defaultConfig.MaxRetryDelay, config.DefaultHttpRetryConfig.MaxRetryDelay); + Assert.Equal(defaultConfig.MinRetryDelay, config.DefaultHttpRetryConfig.MinRetryDelay); + Assert.Equal(defaultConfig.MaxTotalRetryTime, config.DefaultHttpRetryConfig.MaxTotalRetryTime); + Assert.Equal(defaultConfig.UseExponentialBackoff, config.DefaultHttpRetryConfig.UseExponentialBackoff); + Assert.Equal(defaultConfig.RetryableStatusCodes, config.DefaultHttpRetryConfig.RetryableStatusCodes); + Assert.Equal(defaultConfig.RetryableExceptionTypes, config.DefaultHttpRetryConfig.RetryableExceptionTypes); + } + + [Fact] + public void SetDefaultHttpRetryConfigToDefaultIfNull() + { + // Arrange + var config = new KernelConfig(); + + // Act + config.SetDefaultHttpRetryConfig(null); + + // Assert + var defaultConfig = new KernelConfig.HttpRetryConfig(); + Assert.Equal(defaultConfig.MaxRetryCount, config.DefaultHttpRetryConfig.MaxRetryCount); + Assert.Equal(defaultConfig.MaxRetryDelay, config.DefaultHttpRetryConfig.MaxRetryDelay); + Assert.Equal(defaultConfig.MinRetryDelay, config.DefaultHttpRetryConfig.MinRetryDelay); + Assert.Equal(defaultConfig.MaxTotalRetryTime, config.DefaultHttpRetryConfig.MaxTotalRetryTime); + Assert.Equal(defaultConfig.UseExponentialBackoff, config.DefaultHttpRetryConfig.UseExponentialBackoff); + Assert.Equal(defaultConfig.RetryableStatusCodes, config.DefaultHttpRetryConfig.RetryableStatusCodes); + Assert.Equal(defaultConfig.RetryableExceptionTypes, config.DefaultHttpRetryConfig.RetryableExceptionTypes); } [Fact] diff --git a/dotnet/src/SemanticKernel.Test/KernelTests.cs b/dotnet/src/SemanticKernel.Test/KernelTests.cs index 210d3df0e859..253f20514f17 100644 --- a/dotnet/src/SemanticKernel.Test/KernelTests.cs +++ b/dotnet/src/SemanticKernel.Test/KernelTests.cs @@ -139,7 +139,8 @@ public void ItFailsIfCompletionBackendConfigIsNotSet() // Arrange var kernel = KernelBuilder.Create(); - var exception = Assert.Throws(() => kernel.CreateSemanticFunction(promptTemplate: "Tell me a joke", functionName: "joker", skillName: "jk", description: "Nice fun")); + var exception = Assert.Throws(() => + kernel.CreateSemanticFunction(promptTemplate: "Tell me a joke", functionName: "joker", skillName: "jk", description: "Nice fun")); } public class MySkill diff --git a/dotnet/src/SemanticKernel.Test/Reliability/DefaultHttpRetryHandlerTests.cs b/dotnet/src/SemanticKernel.Test/Reliability/DefaultHttpRetryHandlerTests.cs new file mode 100644 index 000000000000..254e1b99ef30 --- /dev/null +++ b/dotnet/src/SemanticKernel.Test/Reliability/DefaultHttpRetryHandlerTests.cs @@ -0,0 +1,642 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Reliability; +using Moq; +using Moq.Protected; +using Xunit; +using static Microsoft.SemanticKernel.Configuration.KernelConfig; + +namespace SemanticKernelTests.Reliability; + +public class DefaultHttpRetryHandlerTests +{ + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + [InlineData(HttpStatusCode.TooManyRequests)] + public async Task NoMaxRetryCountCallsOnceForStatusAsync(HttpStatusCode statusCode) + { + // Arrange + using var retry = new DefaultHttpRetryHandler(new HttpRetryConfig() { MaxRetryCount = 0 }, Mock.Of()); + using var mockResponse = new HttpResponseMessage(statusCode); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + [InlineData(HttpStatusCode.TooManyRequests)] + public async Task ItRetriesOnceOnRetryableStatusAsync(HttpStatusCode statusCode) + { + // Arrange + using var retry = ConfigureRetryHandler(); + using var mockResponse = new HttpResponseMessage(statusCode); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + } + + [Theory] + [InlineData(typeof(TimeoutException))] + [InlineData(typeof(WebException))] + [InlineData(typeof(HttpRequestException))] + public async Task ItRetriesOnceOnRetryableExceptionAsync(Type exceptionType) + { + // Arrange + using var retry = ConfigureRetryHandler(); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(exceptionType); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await Assert.ThrowsAsync(exceptionType, + async () => await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Theory] + [InlineData(typeof(TimeoutException))] + [InlineData(typeof(WebException))] + [InlineData(typeof(HttpRequestException))] + public async Task NoMaxRetryCountCallsOnceForExceptionAsync(Type exceptionType) + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 0 }); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(exceptionType); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await Assert.ThrowsAsync(exceptionType, + async () => await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + [InlineData(HttpStatusCode.TooManyRequests)] + public async Task ItRetriesOnceOnTransientStatusWithExponentialBackoffAsync(HttpStatusCode statusCode) + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { UseExponentialBackoff = true }); + using var mockResponse = new HttpResponseMessage(statusCode); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + } + + [Theory] + [InlineData(typeof(TimeoutException))] + [InlineData(typeof(WebException))] + [InlineData(typeof(HttpRequestException))] + public async Task ItRetriesOnceOnRetryableExceptionWithExponentialBackoffAsync(Type exceptionType) + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { UseExponentialBackoff = true }); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(exceptionType); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await Assert.ThrowsAsync(exceptionType, + async () => await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + public async Task ItRetriesExponentiallyWithExponentialBackoffAsync(HttpStatusCode statusCode) + { + // Arrange + var currentTime = DateTimeOffset.UtcNow; + var mockTimeProvider = new Mock(); + var mockDelayProvider = new Mock(); + mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) + .Returns(() => currentTime) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(5)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(510)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(1015)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(1520)); + using var retry = ConfigureRetryHandler(new HttpRetryConfig() + { + UseExponentialBackoff = true, MaxRetryCount = 3, + MinRetryDelay = TimeSpan.FromMilliseconds(500) + }, mockTimeProvider, mockDelayProvider); + using var mockResponse = new HttpResponseMessage(statusCode); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(4), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(4)); + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(500), It.IsAny()), Times.Once); + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(1000), It.IsAny()), Times.Once); + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(2000), It.IsAny()), Times.Once); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + public async Task ItRetriesOnceOnTransientStatusCodeWithRetryValueAsync(HttpStatusCode statusCode) + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig(), null); + using var mockResponse = new HttpResponseMessage() + { + StatusCode = statusCode, + Headers = { RetryAfter = new RetryConditionHeaderValue(new TimeSpan(0, 0, 0, 1)) }, + }; + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + using var testContent = new StringContent("test"); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + Assert.Equal(new TimeSpan(0, 0, 0, 1), response.Headers.RetryAfter?.Delta); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + public async Task ItRetriesStatusCustomCountAsync(HttpStatusCode expectedStatus) + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 3 }, null); + using var mockResponse = new HttpResponseMessage(expectedStatus); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(4), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(expectedStatus, response.StatusCode); + } + + [Theory] + [InlineData(typeof(TimeoutException))] + [InlineData(typeof(WebException))] + [InlineData(typeof(HttpRequestException))] + public async Task ItRetriesExceptionsCustomCountAsync(Type expectedException) + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 3 }, null); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(expectedException); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await Assert.ThrowsAsync(expectedException, + async () => await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(4), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Fact] + public async Task NoExceptionNoRetryAsync() + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 3 }, null); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.OK); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(1), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + } + + [Fact] + public async Task ItDoesNotExecuteOnCancellationTokenAsync() + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 3 }, null); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.OK); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + var cancellationToken = new CancellationToken(true); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await Assert.ThrowsAsync(async () => + await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, cancellationToken)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Never(), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Fact] + public async Task ItDoestExecuteOnFalseCancellationTokenAsync() + { + // Arrange + using var retry = ConfigureRetryHandler(new HttpRetryConfig() { MaxRetryCount = 3 }, null); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.OK); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + var cancellationToken = new CancellationToken(false); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, cancellationToken); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(1), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + } + + [Fact] + public async Task ItRetriesWithMinRetryDelayAsync() + { + var HttpRetryConfig = new HttpRetryConfig + { + MinRetryDelay = TimeSpan.FromMilliseconds(500) + }; + + var mockDelayProvider = new Mock(); + var mockTimeProvider = new Mock(); + + var currentTime = DateTimeOffset.UtcNow; + + mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) + .Returns(() => currentTime) + .Returns(() => currentTime.AddMilliseconds(5)) + .Returns(() => currentTime.AddMilliseconds(510)); + + mockDelayProvider.Setup(x => x.DelayAsync(It.IsAny(), It.IsAny())) + .Returns(() => Task.CompletedTask); + + using var retry = ConfigureRetryHandler(HttpRetryConfig, mockTimeProvider, mockDelayProvider); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(2)); + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(500), It.IsAny()), Times.Once); + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + } + + [Fact] + public async Task ItRetriesWithMaxRetryDelayAsync() + { + var HttpRetryConfig = new HttpRetryConfig + { + MinRetryDelay = TimeSpan.FromMilliseconds(1), + MaxRetryDelay = TimeSpan.FromMilliseconds(500) + }; + + var mockDelayProvider = new Mock(); + var mockTimeProvider = new Mock(); + + var currentTime = DateTimeOffset.UtcNow; + + mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) + .Returns(() => currentTime) + .Returns(() => currentTime.AddMilliseconds(5)) + .Returns(() => currentTime.AddMilliseconds(505)); + + mockDelayProvider.Setup(x => x.DelayAsync(It.IsAny(), It.IsAny())) + .Returns(() => Task.CompletedTask); + + using var retry = ConfigureRetryHandler(HttpRetryConfig, mockTimeProvider, mockDelayProvider); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests) + { Headers = { RetryAfter = new RetryConditionHeaderValue(TimeSpan.FromMilliseconds(2000)) } }; + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(2)); + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(500), It.IsAny()), Times.Once); + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + Assert.Equal(TimeSpan.FromMilliseconds(2000), response.Headers.RetryAfter?.Delta); + } + + [Theory] + [InlineData(HttpStatusCode.TooManyRequests)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + [InlineData(HttpStatusCode.RequestTimeout)] + public async Task ItRetriesWithMaxTotalDelayAsync(HttpStatusCode statusCode) + { + // Arrange + var HttpRetryConfig = new HttpRetryConfig + { + MaxRetryCount = 5, + MinRetryDelay = TimeSpan.FromMilliseconds(50), + MaxRetryDelay = TimeSpan.FromMilliseconds(50), + MaxTotalRetryTime = TimeSpan.FromMilliseconds(350) + }; + + var mockDelayProvider = new Mock(); + var mockTimeProvider = new Mock(); + + var currentTime = DateTimeOffset.UtcNow; + mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) + .Returns(() => currentTime) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(5)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(55)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(110)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(165)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(220)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(275)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(330)); + + using var retry = ConfigureRetryHandler(HttpRetryConfig, mockTimeProvider, mockDelayProvider); + + using var mockResponse = new HttpResponseMessage(statusCode); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(6)); // one for the initial call, and one for each of 5 attempts + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(50), It.IsAny()), Times.Exactly(5)); + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(6), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(statusCode, response.StatusCode); + } + + [Fact] + public async Task ItRetriesFewerWithMaxTotalDelayAsync() + { + // Arrange + var HttpRetryConfig = new HttpRetryConfig + { + MaxRetryCount = 5, + MinRetryDelay = TimeSpan.FromMilliseconds(50), + MaxRetryDelay = TimeSpan.FromMilliseconds(50), + MaxTotalRetryTime = TimeSpan.FromMilliseconds(100) + }; + + var mockDelayProvider = new Mock(); + var mockTimeProvider = new Mock(); + + var currentTime = DateTimeOffset.UtcNow; + mockTimeProvider.SetupSequence(x => x.GetCurrentTime()) + .Returns(() => currentTime) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(5)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(55)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(110)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(165)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(220)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(275)) + .Returns(() => currentTime + TimeSpan.FromMilliseconds(330)); + + using var retry = ConfigureRetryHandler(HttpRetryConfig, mockTimeProvider, mockDelayProvider); + + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockTimeProvider.Verify(x => x.GetCurrentTime(), Times.Exactly(3)); // one for the initial call, and one for each of 5 attempts + mockDelayProvider.Verify(x => x.DelayAsync(TimeSpan.FromMilliseconds(50), It.IsAny()), Times.Exactly(1)); + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + } + + [Fact] + public async Task ItRetriesOnRetryableStatusCodesAsync() + { + // Arrange + var config = new HttpRetryConfig() { RetryableStatusCodes = new List { HttpStatusCode.Unauthorized } }; + using var retry = ConfigureRetryHandler(config); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.Unauthorized); + + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); + } + + [Fact] + public async Task ItDoesNotRetryOnNonRetryableStatusCodesAsync() + { + // Arrange + var config = new HttpRetryConfig() { RetryableStatusCodes = new List { HttpStatusCode.Unauthorized } }; + using var retry = ConfigureRetryHandler(config); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); + + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + } + + [Fact] + public async Task ItRetriesOnRetryableExceptionsAsync() + { + // Arrange + var config = new HttpRetryConfig() { RetryableExceptionTypes = new List { typeof(InvalidOperationException) } }; + using var retry = ConfigureRetryHandler(config); + + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(typeof(InvalidOperationException)); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + await Assert.ThrowsAsync(async () => + await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Exactly(2), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Fact] + public async Task ItDoesNotRetryOnNonRetryableExceptionsAsync() + { + // Arrange + var config = new HttpRetryConfig() { RetryableExceptionTypes = new List { typeof(InvalidOperationException) } }; + using var retry = ConfigureRetryHandler(config); + + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(typeof(ArgumentException)); + + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + await Assert.ThrowsAsync(async () => + await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + } + + private static DefaultHttpRetryHandler ConfigureRetryHandler(HttpRetryConfig? config = null, + Mock? timeProvider = null, Mock? delayProvider = null) + { + delayProvider ??= new Mock(); + timeProvider ??= new Mock(); + var retry = new DefaultHttpRetryHandler(config ?? new HttpRetryConfig(), Mock.Of(), delayProvider.Object, timeProvider.Object); + return retry; + } + + private static Mock GetHttpMessageHandlerMock(HttpResponseMessage mockResponse) + { + var mockHandler = new Mock(); + mockHandler.Protected() + .Setup>("SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .ReturnsAsync(mockResponse); + return mockHandler; + } + + private static Mock GetHttpMessageHandlerMock(Type exceptionType) + { + var mockHandler = new Mock(); + mockHandler.Protected() + .Setup>("SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .ThrowsAsync(Activator.CreateInstance(exceptionType) as Exception); + return mockHandler; + } +} diff --git a/dotnet/src/SemanticKernel.Test/Reliability/NullHttpRetryHandlerTests.cs b/dotnet/src/SemanticKernel.Test/Reliability/NullHttpRetryHandlerTests.cs new file mode 100644 index 000000000000..8032700e13f7 --- /dev/null +++ b/dotnet/src/SemanticKernel.Test/Reliability/NullHttpRetryHandlerTests.cs @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Reliability; +using Moq; +using Moq.Protected; +using Xunit; + +namespace SemanticKernelTests.Reliability; + +public class NullHttpRetryHandlerTests +{ + [Fact] + public async Task ItDoesNotRetryOnExceptionAsync() + { + // Arrange + using var retry = new NullHttpRetryHandler(); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + } + + [Fact] + public async Task NoExceptionNoRetryAsync() + { + // Arrange + using var retry = new NullHttpRetryHandler(); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.OK); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, CancellationToken.None); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + } + + [Fact] + public async Task TaskCanceledExceptionThrownOnCancellationTokenAsync() + { + // Arrange + using var retry = new NullHttpRetryHandler(); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + using var cancellationTokenSource = new CancellationTokenSource(); + cancellationTokenSource.Cancel(); + + // Act + await Assert.ThrowsAsync(async () => + await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, cancellationTokenSource.Token)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + } + + [Fact] + public async Task ItDoestExecuteOnFalseCancellationTokenAsync() + { + // Arrange + using var retry = new NullHttpRetryHandler(); + using var mockResponse = new HttpResponseMessage(HttpStatusCode.TooManyRequests); + using var testContent = new StringContent("test"); + var mockHandler = GetHttpMessageHandlerMock(mockResponse); + retry.InnerHandler = mockHandler.Object; + using var httpClient = new HttpClient(retry); + + // Act + var response = await httpClient.PostAsync(new Uri("https://www.microsoft.com"), testContent, new CancellationToken(false)); + + // Assert + mockHandler.Protected() + .Verify>("SendAsync", Times.Once(), ItExpr.IsAny(), ItExpr.IsAny()); + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + } + + private static Mock GetHttpMessageHandlerMock(HttpResponseMessage mockResponse) + { + var mockHandler = new Mock(); + mockHandler.Protected() + .Setup>("SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .ReturnsAsync(mockResponse); + return mockHandler; + } +} diff --git a/dotnet/src/SemanticKernel.Test/Reliability/PassThroughWithoutRetryTests.cs b/dotnet/src/SemanticKernel.Test/Reliability/PassThroughWithoutRetryTests.cs deleted file mode 100644 index 85f905d70f77..000000000000 --- a/dotnet/src/SemanticKernel.Test/Reliability/PassThroughWithoutRetryTests.cs +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.AI; -using Microsoft.SemanticKernel.Reliability; -using Moq; -using Xunit; - -namespace SemanticKernelTests.Reliability; - -public class PassThroughWithoutRetryTests -{ - [Fact] - public async Task ItDoesNotRetryOnExceptionAsync() - { - // Arrange - var retry = new PassThroughWithoutRetry(); - var action = new Mock>(); - action.Setup(a => a()).Throws(new AIException(AIException.ErrorCodes.Throttling, "Throttling Test")); - - // Act - await Assert.ThrowsAsync(() => retry.ExecuteWithRetryAsync(action.Object, Mock.Of())); - - // Assert - action.Verify(a => a(), Times.Once); - } - - [Fact] - public async Task NoExceptionNoRetryAsync() - { - // Arrange - var log = new Mock(); - var retry = new PassThroughWithoutRetry(); - var action = new Mock>(); - - // Act - await retry.ExecuteWithRetryAsync(action.Object, log.Object); - - // Assert - action.Verify(a => a(), Times.Once); - } - - [Fact] - public async Task ItDoesNotExecuteOnCancellationTokenAsync() - { - // Arrange - var retry = new PassThroughWithoutRetry(); - var action = new Mock>(); - action.Setup(a => a()).Throws(new AIException(AIException.ErrorCodes.Throttling, "Throttling Test")); - - // Act - await retry.ExecuteWithRetryAsync(action.Object, Mock.Of(), new CancellationToken(true)); - - // Assert - action.Verify(a => a(), Times.Never); - } - - [Fact] - public async Task ItDoestExecuteOnFalseCancellationTokenAsync() - { - // Arrange - var retry = new PassThroughWithoutRetry(); - var action = new Mock>(); - action.Setup(a => a()).Throws(new AIException(AIException.ErrorCodes.Throttling, "Throttling Test")); - - // Act - await Assert.ThrowsAsync(() => retry.ExecuteWithRetryAsync(action.Object, Mock.Of(), new CancellationToken(false))); - - // Assert - action.Verify(a => a(), Times.Once); - } -} diff --git a/dotnet/src/SemanticKernel/AI/ITextCompletionClient.cs b/dotnet/src/SemanticKernel/AI/ITextCompletionClient.cs index a4118375d8db..833c6b495c6f 100644 --- a/dotnet/src/SemanticKernel/AI/ITextCompletionClient.cs +++ b/dotnet/src/SemanticKernel/AI/ITextCompletionClient.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Threading; using System.Threading.Tasks; namespace Microsoft.SemanticKernel.AI; @@ -14,6 +15,7 @@ public interface ITextCompletionClient /// /// The prompt to complete. /// Request settings for the completion API + /// Cancellation token /// Text generated by the remote model - public Task CompleteAsync(string text, CompleteRequestSettings requestSettings); + public Task CompleteAsync(string text, CompleteRequestSettings requestSettings, CancellationToken cancellationToken = default); } diff --git a/dotnet/src/SemanticKernel/AI/OpenAI/Clients/AzureOpenAIClientAbstract.cs b/dotnet/src/SemanticKernel/AI/OpenAI/Clients/AzureOpenAIClientAbstract.cs index f3ab80244db7..a7e3450ec2bb 100644 --- a/dotnet/src/SemanticKernel/AI/OpenAI/Clients/AzureOpenAIClientAbstract.cs +++ b/dotnet/src/SemanticKernel/AI/OpenAI/Clients/AzureOpenAIClientAbstract.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel.AI.OpenAI.HttpSchema; +using Microsoft.SemanticKernel.Reliability; using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.AI.OpenAI.Clients; @@ -23,6 +24,7 @@ public abstract class AzureOpenAIClientAbstract : OpenAIClientAbstract /// /// Azure OpenAI API version /// + /// protected string AzureOpenAIApiVersion { get { return this._azureOpenAIApiVersion; } @@ -48,7 +50,8 @@ protected string AzureOpenAIApiVersion /// Construct an AzureOpenAIClientAbstract object /// /// Logger - protected AzureOpenAIClientAbstract(ILogger? log = null) : base(log) + /// Retry handler factory + protected AzureOpenAIClientAbstract(ILogger? log = null, IDelegatingHandlerFactory? handlerFactory = null) : base(log, handlerFactory) { } diff --git a/dotnet/src/SemanticKernel/AI/OpenAI/Clients/OpenAIClientAbstract.cs b/dotnet/src/SemanticKernel/AI/OpenAI/Clients/OpenAIClientAbstract.cs index 869cb9de2e41..874d0ed59603 100644 --- a/dotnet/src/SemanticKernel/AI/OpenAI/Clients/OpenAIClientAbstract.cs +++ b/dotnet/src/SemanticKernel/AI/OpenAI/Clients/OpenAIClientAbstract.cs @@ -7,11 +7,13 @@ using System.Net.Http; using System.Net.Mime; using System.Text; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.SemanticKernel.AI.Embeddings; using Microsoft.SemanticKernel.AI.OpenAI.HttpSchema; +using Microsoft.SemanticKernel.Reliability; using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.AI.OpenAI.Clients; @@ -33,14 +35,19 @@ public abstract class OpenAIClientAbstract : IDisposable protected HttpClient HTTPClient { get; } private readonly HttpClientHandler _httpClientHandler; + private readonly IDelegatingHandlerFactory _handlerFactory = new DefaultHttpRetryHandlerFactory(); + private readonly DelegatingHandler _retryHandler; - internal OpenAIClientAbstract(ILogger? log = null) + internal OpenAIClientAbstract(ILogger? log = null, IDelegatingHandlerFactory? handlerFactory = null) { - if (log != null) { this.Log = log; } + this.Log = log ?? this.Log; + this._handlerFactory = handlerFactory ?? this._handlerFactory; - // TODO: allow injection of retry logic, e.g. Polly this._httpClientHandler = new() { CheckCertificateRevocationList = true }; - this.HTTPClient = new HttpClient(this._httpClientHandler); + this._retryHandler = this._handlerFactory.Create(); + this._retryHandler.InnerHandler = this._httpClientHandler; + + this.HTTPClient = new HttpClient(this._retryHandler); this.HTTPClient.DefaultRequestHeaders.Add("User-Agent", HTTPUseragent); } @@ -49,15 +56,16 @@ internal OpenAIClientAbstract(ILogger? log = null) /// /// URL for the completion request API /// Prompt to complete + /// Cancellation token /// The completed text /// AIException thrown during the request. - protected async Task ExecuteCompleteRequestAsync(string url, string requestBody) + protected async Task ExecuteCompleteRequestAsync(string url, string requestBody, CancellationToken cancellationToken = default) { try { this.Log.LogDebug("Sending completion request to {0}: {1}", url, requestBody); - var result = await this.ExecutePostRequestAsync(url, requestBody); + var result = await this.ExecutePostRequestAsync(url, requestBody, cancellationToken); if (result.Completions.Count < 1) { throw new AIException( @@ -124,6 +132,7 @@ protected virtual void Dispose(bool disposing) { this.HTTPClient.Dispose(); this._httpClientHandler.Dispose(); + this._retryHandler.Dispose(); } } @@ -132,14 +141,15 @@ protected virtual void Dispose(bool disposing) // HTTP user agent sent to remote endpoints private const string HTTPUseragent = "Microsoft Semantic Kernel"; - private async Task ExecutePostRequestAsync(string url, string requestBody) + private async Task ExecutePostRequestAsync(string url, string requestBody, CancellationToken cancellationToken = default) { string responseJson; try { using HttpContent content = new StringContent(requestBody, Encoding.UTF8, MediaTypeNames.Application.Json); - HttpResponseMessage response = await this.HTTPClient.PostAsync(url, content); + + HttpResponseMessage response = await this.HTTPClient.PostAsync(url, content, cancellationToken); if (response == null) { diff --git a/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextCompletion.cs b/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextCompletion.cs index 38e8055a47c4..aa4cbff440d9 100644 --- a/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextCompletion.cs +++ b/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextCompletion.cs @@ -1,10 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel.AI.OpenAI.Clients; using Microsoft.SemanticKernel.AI.OpenAI.HttpSchema; using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Reliability; using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.AI.OpenAI.Services; @@ -22,8 +24,10 @@ public sealed class AzureTextCompletion : AzureOpenAIClientAbstract, ITextComple /// Azure OpenAI API key, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// Azure OpenAI API version, see https://learn.microsoft.com/azure/cognitive-services/openai/reference /// Application logger - public AzureTextCompletion(string modelId, string endpoint, string apiKey, string apiVersion, ILogger? log = null) - : base(log) + /// Retry handler factory for HTTP requests. + public AzureTextCompletion(string modelId, string endpoint, string apiKey, string apiVersion, ILogger? log = null, + IDelegatingHandlerFactory? handlerFactory = null) + : base(log, handlerFactory) { Verify.NotEmpty(modelId, "The ID cannot be empty, you must provide a Model ID or a Deployment name."); this._modelId = modelId; @@ -43,9 +47,10 @@ public AzureTextCompletion(string modelId, string endpoint, string apiKey, strin /// /// Text to complete /// Request settings for the completion API + /// Cancellation token /// The completed text. /// AIException thrown during the request - public async Task CompleteAsync(string text, CompleteRequestSettings requestSettings) + public async Task CompleteAsync(string text, CompleteRequestSettings requestSettings, CancellationToken cancellationToken = default) { Verify.NotNull(requestSettings, "Completion settings cannot be empty"); @@ -72,7 +77,7 @@ public async Task CompleteAsync(string text, CompleteRequestSettings req Stop = requestSettings.StopSequences is { Count: > 0 } ? requestSettings.StopSequences : null, }); - return await this.ExecuteCompleteRequestAsync(url, requestBody); + return await this.ExecuteCompleteRequestAsync(url, requestBody, cancellationToken); } #region private ================================================================================ diff --git a/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextEmbeddings.cs b/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextEmbeddings.cs index e7c2dfac1a51..283d56d5b3db 100644 --- a/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextEmbeddings.cs +++ b/dotnet/src/SemanticKernel/AI/OpenAI/Services/AzureTextEmbeddings.cs @@ -7,6 +7,7 @@ using Microsoft.SemanticKernel.AI.OpenAI.Clients; using Microsoft.SemanticKernel.AI.OpenAI.HttpSchema; using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Reliability; using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.AI.OpenAI.Services; @@ -26,8 +27,10 @@ public sealed class AzureTextEmbeddings : AzureOpenAIClientAbstract, IEmbeddingG /// Azure OpenAI API key, see https://learn.microsoft.com/azure/cognitive-services/openai/quickstart /// Azure OpenAI API version, see https://learn.microsoft.com/azure/cognitive-services/openai/reference /// Application logger - public AzureTextEmbeddings(string modelId, string endpoint, string apiKey, string apiVersion, ILogger? log = null) - : base(log) + /// An optional HTTP retry handler factory + public AzureTextEmbeddings(string modelId, string endpoint, string apiKey, string apiVersion, ILogger? log = null, + IDelegatingHandlerFactory? handlerFactory = null) + : base(log, handlerFactory) { Verify.NotEmpty(modelId, "The ID cannot be empty, you must provide a Model ID or a Deployment name."); this._modelId = modelId; diff --git a/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextCompletion.cs b/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextCompletion.cs index 00f6553f7ef2..95ec72da38da 100644 --- a/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextCompletion.cs +++ b/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextCompletion.cs @@ -1,11 +1,13 @@ // Copyright (c) Microsoft. All rights reserved. using System.Net.Http.Headers; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel.AI.OpenAI.Clients; using Microsoft.SemanticKernel.AI.OpenAI.HttpSchema; using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Reliability; using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.AI.OpenAI.Services; @@ -27,8 +29,10 @@ public sealed class OpenAITextCompletion : OpenAIClientAbstract, ITextCompletion /// OpenAI API key, see https://platform.openai.com/account/api-keys /// OpenAI organization id. This is usually optional unless your account belongs to multiple organizations. /// Logger - public OpenAITextCompletion(string modelId, string apiKey, string? organization = null, ILogger? log = null) : - base(log) + /// Retry handler + public OpenAITextCompletion(string modelId, string apiKey, string? organization = null, ILogger? log = null, + IDelegatingHandlerFactory? handlerFactory = null) : + base(log, handlerFactory) { Verify.NotEmpty(modelId, "The OpenAI model ID cannot be empty"); this._modelId = modelId; @@ -47,9 +51,10 @@ public OpenAITextCompletion(string modelId, string apiKey, string? organization /// /// The prompt to complete. /// Request settings for the completion API + /// Cancellation token /// The completed text /// AIException thrown during the request - public async Task CompleteAsync(string text, CompleteRequestSettings requestSettings) + public async Task CompleteAsync(string text, CompleteRequestSettings requestSettings, CancellationToken cancellationToken = default) { Verify.NotNull(requestSettings, "Completion settings cannot be empty"); @@ -74,6 +79,6 @@ public async Task CompleteAsync(string text, CompleteRequestSettings req Stop = requestSettings.StopSequences is { Count: > 0 } ? requestSettings.StopSequences : null, }); - return await this.ExecuteCompleteRequestAsync(url, requestBody); + return await this.ExecuteCompleteRequestAsync(url, requestBody, cancellationToken); } } diff --git a/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextEmbeddings.cs b/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextEmbeddings.cs index 6506f4710620..039f0d937c57 100644 --- a/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextEmbeddings.cs +++ b/dotnet/src/SemanticKernel/AI/OpenAI/Services/OpenAITextEmbeddings.cs @@ -8,6 +8,7 @@ using Microsoft.SemanticKernel.AI.OpenAI.Clients; using Microsoft.SemanticKernel.AI.OpenAI.HttpSchema; using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Reliability; using Microsoft.SemanticKernel.Text; namespace Microsoft.SemanticKernel.AI.OpenAI.Services; @@ -30,8 +31,10 @@ public sealed class OpenAITextEmbeddings : OpenAIClientAbstract, IEmbeddingGener /// OpenAI API Key /// Optional OpenAI organization ID, usually required only if your account belongs to multiple organizations /// Application logger - public OpenAITextEmbeddings(string modelId, string apiKey, string? organization = null, ILogger? log = null) - : base(log) + /// Retry handler factory for HTTP requests. + public OpenAITextEmbeddings(string modelId, string apiKey, string? organization = null, ILogger? log = null, + IDelegatingHandlerFactory? handlerFactory = null) + : base(log, handlerFactory) { Verify.NotEmpty(modelId, "The OpenAI model ID cannot be empty"); this._modelId = modelId; diff --git a/dotnet/src/SemanticKernel/Configuration/KernelConfig.cs b/dotnet/src/SemanticKernel/Configuration/KernelConfig.cs index ef8cf8cff877..2617b621b27f 100644 --- a/dotnet/src/SemanticKernel/Configuration/KernelConfig.cs +++ b/dotnet/src/SemanticKernel/Configuration/KernelConfig.cs @@ -3,6 +3,9 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Net; +using System.Net.Http; +using Microsoft.Extensions.Logging.Abstractions; using Microsoft.SemanticKernel.AI.OpenAI.Services; using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Reliability; @@ -15,9 +18,78 @@ namespace Microsoft.SemanticKernel.Configuration; public sealed class KernelConfig { /// - /// Global retry logic used for all the backends + /// Retry configuration for IHttpRetryPolicy that uses RetryAfter header when present. /// - public IRetryMechanism RetryMechanism { get => this._retryMechanism; } + public sealed class HttpRetryConfig + { + /// + /// Maximum number of retries. + /// + /// Thrown when value is negative. + public int MaxRetryCount + { + get { return this._maxRetryCount; } + set + { + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(this.MaxRetryCount), "Max retry count cannot be negative."); + } + + this._maxRetryCount = value; + } + } + + /// + /// Minimum delay between retries. + /// + public TimeSpan MinRetryDelay { get; set; } = TimeSpan.FromSeconds(2); + + /// + /// Maximum delay between retries. + /// + public TimeSpan MaxRetryDelay { get; set; } = TimeSpan.FromSeconds(60); + + /// + /// Maximum total time spent retrying. + /// + public TimeSpan MaxTotalRetryTime { get; set; } = TimeSpan.FromMinutes(2); + + /// + /// Whether to use exponential backoff or not. + /// + public bool UseExponentialBackoff { get; set; } + + /// + /// List of status codes that should be retried. + /// + public List RetryableStatusCodes { get; set; } = new() + { + HttpStatusCode.RequestTimeout, + HttpStatusCode.ServiceUnavailable, + HttpStatusCode.GatewayTimeout, + HttpStatusCode.TooManyRequests + }; + + /// + /// List of exception types that should be retried. + /// + public List RetryableExceptionTypes { get; set; } = new() + { + typeof(TimeoutException), + typeof(WebException), + typeof(HttpRequestException) + }; + + private int _maxRetryCount = 1; + } + + /// + /// Global retry logic used for all the backends http calls + /// + public IDelegatingHandlerFactory HttpHandlerFactory { get; private set; } = new DefaultHttpRetryHandlerFactory(new HttpRetryConfig(), NullLogger.Instance); + + public HttpRetryConfig DefaultHttpRetryConfig { get; private set; } = new(); /// /// Adds an Azure OpenAI backend to the list. @@ -180,13 +252,27 @@ public bool HasEmbeddingsBackend(string label, Func? condi } /// - /// Set the retry mechanism to use for the kernel. + /// Set the http retry handler factory to use for the kernel. /// - /// Retry mechanism to use. + /// Retry handler factory to use. /// The updated kernel configuration. - public KernelConfig SetRetryMechanism(IRetryMechanism? retryMechanism = null) + public KernelConfig SetHttpHandlerFactory(IDelegatingHandlerFactory? httpHandlerFactory = null) + { + if (httpHandlerFactory != null) + { + this.HttpHandlerFactory = httpHandlerFactory; + } + + return this; + } + + public KernelConfig SetDefaultHttpRetryConfig(HttpRetryConfig? httpRetryConfig) { - this._retryMechanism = retryMechanism ?? new PassThroughWithoutRetry(); + if (httpRetryConfig != null) + { + this.DefaultHttpRetryConfig = httpRetryConfig; + } + return this; } @@ -397,7 +483,6 @@ public KernelConfig RemoveAllBackends() private Dictionary EmbeddingsBackends { get; set; } = new(); private string? _defaultCompletionBackend; private string? _defaultEmbeddingsBackend; - private IRetryMechanism _retryMechanism = new PassThroughWithoutRetry(); #endregion } diff --git a/dotnet/src/SemanticKernel/Kernel.cs b/dotnet/src/SemanticKernel/Kernel.cs index 5bf11bbba4b1..a90214b31326 100644 --- a/dotnet/src/SemanticKernel/Kernel.cs +++ b/dotnet/src/SemanticKernel/Kernel.cs @@ -172,10 +172,7 @@ public async Task RunAsync(ContextVariables variables, CancellationTo try { cancellationToken.ThrowIfCancellationRequested(); - await this._config.RetryMechanism.ExecuteWithRetryAsync( - async () => { context = await f.InvokeAsync(context); }, - this._log, - cancellationToken); + context = await f.InvokeAsync(context); if (context.ErrorOccurred) { @@ -268,7 +265,8 @@ private ISKFunction CreateSemanticFunction( azureBackendConfig.Endpoint, azureBackendConfig.APIKey, azureBackendConfig.APIVersion, - this._log)); + this._log, + this._config.HttpHandlerFactory)); break; case OpenAIConfig openAiConfig: @@ -276,7 +274,8 @@ private ISKFunction CreateSemanticFunction( openAiConfig.ModelId, openAiConfig.APIKey, openAiConfig.OrgId, - this._log)); + this._log, + this._config.HttpHandlerFactory)); break; default: diff --git a/dotnet/src/SemanticKernel/KernelBuilder.cs b/dotnet/src/SemanticKernel/KernelBuilder.cs index d039eb10e671..14f724c0c546 100644 --- a/dotnet/src/SemanticKernel/KernelBuilder.cs +++ b/dotnet/src/SemanticKernel/KernelBuilder.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.Net.Http; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.SemanticKernel.AI.Embeddings; @@ -8,6 +9,7 @@ using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.KernelExtensions; using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Reliability; using Microsoft.SemanticKernel.SkillDefinition; using Microsoft.SemanticKernel.TemplateEngine; @@ -23,6 +25,7 @@ public sealed class KernelBuilder private ISemanticTextMemory _memory = NullMemory.Instance; private ILogger _log = NullLogger.Instance; private IMemoryStore? _memoryStorage = null; + private IDelegatingHandlerFactory? _httpHandlerFactory = null; /// /// Create a new kernel instance @@ -40,6 +43,8 @@ public static IKernel Create() /// Kernel instance public IKernel Build() { + this._config.SetHttpHandlerFactory(this._httpHandlerFactory ?? new DefaultHttpRetryHandlerFactory(this._config.DefaultHttpRetryConfig, this._log)); + var instance = new Kernel( new SkillCollection(this._log), new PromptTemplateEngine(this._log), @@ -108,6 +113,13 @@ public KernelBuilder WithMemoryStorageAndEmbeddingGenerator( return this; } + public KernelBuilder WithRetryHandler(IDelegatingHandlerFactory httpHandlerFactory) + { + Verify.NotNull(httpHandlerFactory, "The retry handler factory instance provided is NULL"); + this._httpHandlerFactory = httpHandlerFactory; + return this; + } + /// /// Use the given configuration with the kernel to be built. /// diff --git a/dotnet/src/SemanticKernel/KernelExtensions/MemoryConfiguration.cs b/dotnet/src/SemanticKernel/KernelExtensions/MemoryConfiguration.cs index eb54f57f5056..76430561e22c 100644 --- a/dotnet/src/SemanticKernel/KernelExtensions/MemoryConfiguration.cs +++ b/dotnet/src/SemanticKernel/KernelExtensions/MemoryConfiguration.cs @@ -51,7 +51,8 @@ public static void UseMemory(this IKernel kernel, string? embeddingsBackendLabel azureAIConfig.Endpoint, azureAIConfig.APIKey, azureAIConfig.APIVersion, - kernel.Log); + kernel.Log, + kernel.Config.HttpHandlerFactory); break; case OpenAIConfig openAIConfig: @@ -59,7 +60,8 @@ public static void UseMemory(this IKernel kernel, string? embeddingsBackendLabel openAIConfig.ModelId, openAIConfig.APIKey, openAIConfig.OrgId, - kernel.Log); + kernel.Log, + kernel.Config.HttpHandlerFactory); break; default: diff --git a/dotnet/src/SemanticKernel/Orchestration/SKFunction.cs b/dotnet/src/SemanticKernel/Orchestration/SKFunction.cs index f5b29153419c..b7061f07d9c3 100644 --- a/dotnet/src/SemanticKernel/Orchestration/SKFunction.cs +++ b/dotnet/src/SemanticKernel/Orchestration/SKFunction.cs @@ -109,7 +109,7 @@ async Task LocalFunc( { string prompt = await functionConfig.PromptTemplate.RenderAsync(context); - string completion = await client.CompleteAsync(prompt, requestSettings); + string completion = await client.CompleteAsync(prompt, requestSettings, context.CancellationToken); context.Variables.Update(completion); } #pragma warning disable CA1031 // We need to catch all exceptions to handle the execution state diff --git a/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandler.cs b/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandler.cs new file mode 100644 index 000000000000..1e29e93a1d7e --- /dev/null +++ b/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandler.cs @@ -0,0 +1,256 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.Configuration; + +namespace Microsoft.SemanticKernel.Reliability; + +internal sealed class DefaultHttpRetryHandler : DelegatingHandler +{ + /// + /// Initializes a new instance of the class. + /// + /// The retry configuration. + /// The logger. + public DefaultHttpRetryHandler(KernelConfig.HttpRetryConfig? config = null, ILogger? log = null) : this(config ?? new KernelConfig.HttpRetryConfig(), log, + null, null) + { + } + + public readonly Guid Id = Guid.NewGuid(); + + internal DefaultHttpRetryHandler(KernelConfig.HttpRetryConfig config, ILogger? log = null, IDelayProvider? delayProvider = null, + ITimeProvider? timeProvider = null) + { + this._config = config; + this._log = log ?? NullLogger.Instance; + this._delayProvider = delayProvider ?? new TaskDelayProvider(); + this._timeProvider = timeProvider ?? new DefaultTimeProvider(); + } + + /// + /// Executes the action with retry logic + /// + /// + /// The request is retried if it throws an exception that is a retryable exception. + /// If the request throws an exception that is not a retryable exception, it is not retried. + /// If the request returns a response with a retryable error code, it is retried. + /// If the request returns a response with a non-retryable error code, it is not retried. + /// If the exception contains a RetryAfter header, the request is retried after the specified delay. + /// If configured to use exponential backoff, the delay is doubled for each retry. + /// + /// The request. + /// The cancellation token. + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + int retryCount = 0; + + var start = this._timeProvider.GetCurrentTime(); + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + TimeSpan waitFor = default; + string reason = string.Empty; + HttpResponseMessage? response = null; + try + { + response = await base.SendAsync(request, cancellationToken); + + // If the request does not require a retry then we're done + if (!this.ShouldRetry(response.StatusCode)) + { + return response; + } + + // Drain response content to free connections. Need to perform this + // before retry attempt and before the TooManyRetries ServiceException. + if (response.Content != null) + { +#if NET5_0_OR_GREATER + await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); +#else + await response.Content.ReadAsByteArrayAsync().ConfigureAwait(false); +#endif + } + + reason = response.StatusCode.ToString(); + + if (retryCount >= this._config.MaxRetryCount) + { + this._log.LogWarning( + "Error executing request, max retry count reached. Reason: {0}", reason); + return response; + } + + // If the retry delay is longer than the total timeout, then we'll + // just return + if (!this.HasTimeForRetry(start, retryCount, response, out waitFor)) + { + this._log.LogWarning( + "Error executing request, max total retry time reached. Reason: {0}", reason); + return response; + } + } + catch (Exception e) when ((this.ShouldRetry(e) || this.ShouldRetry(e.InnerException)) && + retryCount < this._config.MaxRetryCount && + this.HasTimeForRetry(start, retryCount, response, out waitFor)) + { + reason = e.GetType().ToString(); + } + + // If the request requires a retry then we'll retry + this._log.LogWarning( + "Error executing action [attempt {0} of {1}]. Reason: {2}. Will retry after {3}ms", + retryCount + 1, + this._config.MaxRetryCount, + reason, + waitFor.TotalMilliseconds); + + // Clone request with CloneAsync before retrying + // Do not dispose this request as that breaks the request cloning +#pragma warning disable CA2000 + request = await CloneAsync(request); +#pragma warning restore CA2000 + + // Increase retryCount + retryCount++; + + // Delay + await this._delayProvider.DelayAsync(waitFor, cancellationToken).ConfigureAwait(false); + } + } + + private TimeSpan GetWaitTime(int retryCount, HttpResponseMessage? response) + { + var retryAfter = response?.Headers.RetryAfter?.Date.HasValue == true + ? response?.Headers.RetryAfter?.Date - DateTimeOffset.Now + : (response?.Headers.RetryAfter?.Delta) ?? this._config.MinRetryDelay; + retryAfter ??= this._config.MinRetryDelay; + + var timeToWait = retryAfter > this._config.MaxRetryDelay + ? this._config.MaxRetryDelay + : retryAfter < this._config.MinRetryDelay + ? this._config.MinRetryDelay + : retryAfter ?? default; + + if (this._config.UseExponentialBackoff) + { + for (var backoffRetryCount = 1; backoffRetryCount < retryCount + 1; backoffRetryCount++) + { + timeToWait = timeToWait.Add(timeToWait); + } + } + + return timeToWait; + } + + private bool HasTimeForRetry(DateTimeOffset start, int retryCount, HttpResponseMessage? response, out TimeSpan waitFor) + { + waitFor = this.GetWaitTime(retryCount, response); + var currentTIme = this._timeProvider.GetCurrentTime(); + var result = currentTIme - start + waitFor; + + return result < this._config.MaxTotalRetryTime; + } + + private bool ShouldRetry(HttpStatusCode statusCode) + { + return this._config.RetryableStatusCodes.Contains(statusCode); + } + + private bool ShouldRetry(Exception exception) + { + return this._config.RetryableExceptionTypes.Contains(exception.GetType()); + } + + /// + /// Create a new HTTP request by copying previous HTTP request's headers and properties from response's request message. + /// Copied from: https://github.com/microsoftgraph/msgraph-sdk-dotnet-core/blob/dev/src/Microsoft.Graph.Core/Extensions/HttpRequestMessageExtensions.cs + /// + /// The previous needs to be copy. + /// The . + /// + /// Re-issue a new HTTP request with the previous request's headers and properities + /// + internal static async Task CloneAsync(HttpRequestMessage originalRequest) + { + var newRequest = new HttpRequestMessage(originalRequest.Method, originalRequest.RequestUri); + + // Copy request headers. + foreach (var header in originalRequest.Headers) + { + newRequest.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + + // Copy request properties. + foreach (var property in originalRequest.Properties) + { + newRequest.Properties.Add(property); + } + + // Set Content if previous request had one. + if (originalRequest.Content != null) + { + // HttpClient doesn't rewind streams and we have to explicitly do so. + await originalRequest.Content.ReadAsStreamAsync().ContinueWith(t => + { + if (t.Result.CanSeek) + { + t.Result.Seek(0, SeekOrigin.Begin); + } + + newRequest.Content = new StreamContent(t.Result); + }, TaskScheduler.Current).ConfigureAwait(false); + + // Copy content headers. + if (originalRequest.Content.Headers != null) + { + foreach (var contentHeader in originalRequest.Content.Headers) + { + newRequest.Content.Headers.TryAddWithoutValidation(contentHeader.Key, contentHeader.Value); + } + } + } + + return newRequest; + } + + private readonly KernelConfig.HttpRetryConfig _config; + private readonly ILogger _log; + private readonly IDelayProvider _delayProvider; + private readonly ITimeProvider _timeProvider; + + internal interface IDelayProvider + { + Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken); + } + + internal class TaskDelayProvider : IDelayProvider + { + public async Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken) + { + await Task.Delay(delay, cancellationToken); + } + } + + internal interface ITimeProvider + { + DateTimeOffset GetCurrentTime(); + } + + internal class DefaultTimeProvider : ITimeProvider + { + public DateTimeOffset GetCurrentTime() + { + return DateTimeOffset.UtcNow; + } + } +} diff --git a/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandlerFactory.cs b/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandlerFactory.cs new file mode 100644 index 000000000000..e7673261d4f8 --- /dev/null +++ b/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandlerFactory.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Configuration; + +namespace Microsoft.SemanticKernel.Reliability; + +internal class DefaultHttpRetryHandlerFactory : IDelegatingHandlerFactory +{ + internal DefaultHttpRetryHandlerFactory(KernelConfig.HttpRetryConfig? config = null, ILogger? log = null) + { + this._config = config; + this._log = log; + } + + public DelegatingHandler Create() + { + return new DefaultHttpRetryHandler(this._config, this._log); + } + + private readonly KernelConfig.HttpRetryConfig? _config; + private readonly ILogger? _log; +} diff --git a/dotnet/src/SemanticKernel/Reliability/IDelegatingHandlerFactory.cs b/dotnet/src/SemanticKernel/Reliability/IDelegatingHandlerFactory.cs new file mode 100644 index 000000000000..fdf9b1123745 --- /dev/null +++ b/dotnet/src/SemanticKernel/Reliability/IDelegatingHandlerFactory.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; + +namespace Microsoft.SemanticKernel.Reliability; + +/// +/// Factory for creating instances. +/// +public interface IDelegatingHandlerFactory +{ + DelegatingHandler Create(); +} diff --git a/dotnet/src/SemanticKernel/Reliability/IRetryMechanism.cs b/dotnet/src/SemanticKernel/Reliability/IRetryMechanism.cs deleted file mode 100644 index cd92fc8e278c..000000000000 --- a/dotnet/src/SemanticKernel/Reliability/IRetryMechanism.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; - -namespace Microsoft.SemanticKernel.Reliability; - -/// -/// Interface for retry mechanisms on AI calls. -/// -public interface IRetryMechanism -{ - /// - /// Executes the given action with retry logic. - /// - /// The action to retry on exception. - /// The logger to use. - /// The cancellation token. - /// An awaitable task. - Task ExecuteWithRetryAsync(Func action, ILogger log, CancellationToken cancellationToken = default); -} diff --git a/dotnet/src/SemanticKernel/Reliability/NullHttpRetryHandler.cs b/dotnet/src/SemanticKernel/Reliability/NullHttpRetryHandler.cs new file mode 100644 index 000000000000..b8f5988b1237 --- /dev/null +++ b/dotnet/src/SemanticKernel/Reliability/NullHttpRetryHandler.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; + +namespace Microsoft.SemanticKernel.Reliability; + +public class NullHttpRetryHandlerFactory : IDelegatingHandlerFactory +{ + public DelegatingHandler Create() + { + return new NullHttpRetryHandler(); + } +} + +/// +/// A http retry handler that does not retry. +/// +public class NullHttpRetryHandler : DelegatingHandler +{ +} diff --git a/dotnet/src/SemanticKernel/Reliability/PassThroughWithoutRetry.cs b/dotnet/src/SemanticKernel/Reliability/PassThroughWithoutRetry.cs deleted file mode 100644 index 8680722dae04..000000000000 --- a/dotnet/src/SemanticKernel/Reliability/PassThroughWithoutRetry.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.Diagnostics; - -namespace Microsoft.SemanticKernel.Reliability; - -/// -/// A retry mechanism that does not retry. -/// -internal class PassThroughWithoutRetry : IRetryMechanism -{ - public async Task ExecuteWithRetryAsync(Func action, ILogger log, CancellationToken cancellationToken = default) - { - try - { - if (!cancellationToken.IsCancellationRequested) - { - await action(); - } - } - catch (Exception ex) when (!ex.IsCriticalException()) - { - log.LogWarning(ex, "Error executing action, not retrying"); - throw; - } - } -} diff --git a/dotnet/src/SemanticKernel/SemanticKernel.csproj b/dotnet/src/SemanticKernel/SemanticKernel.csproj index e22aa0fba3ce..fb9b247d3912 100644 --- a/dotnet/src/SemanticKernel/SemanticKernel.csproj +++ b/dotnet/src/SemanticKernel/SemanticKernel.csproj @@ -39,5 +39,8 @@ <_Parameter1>SemanticKernelTests + + <_Parameter1>DynamicProxyGenAssembly2 + \ No newline at end of file diff --git a/samples/dotnet/KernelBuilder/Program.cs b/samples/dotnet/KernelBuilder/Program.cs index c542a2dfe811..764381fe66af 100644 --- a/samples/dotnet/KernelBuilder/Program.cs +++ b/samples/dotnet/KernelBuilder/Program.cs @@ -104,15 +104,33 @@ // AI requests (when using the kernel). var kernel8 = Kernel.Builder - .Configure(c => c.SetRetryMechanism(new RetryThreeTimes())) + .Configure(c => c.SetHttpHandlerFactory(new RetryThreeTimesFactory())) .Build(); -public class RetryThreeTimes : IRetryMechanism +public class RetryThreeTimesFactory : IDelegatingHandlerFactory { - public Task ExecuteWithRetryAsync(Func action, ILogger log, CancellationToken cancellationToken = default) + public DelegatingHandler Create() { - var policy = GetPolicy(log); - return policy.ExecuteAsync((_) => action(), cancellationToken); + return new RetryThreeTimes(); + } +} + +public class RetryThreeTimes : DelegatingHandler +{ + private readonly AsyncRetryPolicy _policy; + + public RetryThreeTimes(ILogger log = null) + { + this._policy = GetPolicy(log ?? NullLogger.Instance); + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + return await this._policy.ExecuteAsync(async () => + { + var response = await base.SendAsync(request, cancellationToken); + return response; + }); } private static AsyncRetryPolicy GetPolicy(ILogger log) @@ -126,7 +144,7 @@ private static AsyncRetryPolicy GetPolicy(ILogger log) TimeSpan.FromSeconds(8) }, (ex, timespan, retryCount, _) => log.LogWarning(ex, - "Error executing action [attempt {0} of ], pausing {1} msecs", + "Error executing action [attempt {0} of 3], pausing {1} msecs", retryCount, timespan.TotalMilliseconds)); } } diff --git a/samples/dotnet/kernel-syntax-examples/Example08_RetryHandler.cs b/samples/dotnet/kernel-syntax-examples/Example08_RetryHandler.cs new file mode 100644 index 000000000000..fcd64a20c8fe --- /dev/null +++ b/samples/dotnet/kernel-syntax-examples/Example08_RetryHandler.cs @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Configuration; +using Microsoft.SemanticKernel.CoreSkills; +using Microsoft.SemanticKernel.KernelExtensions; +using Microsoft.SemanticKernel.Reliability; +using Reliability; +using RepoUtils; + +// ReSharper disable once InconsistentNaming +public static class Example08_RetryHandler +{ + public static async Task RunAsync() + { + var kernel = InitializeKernel(); + var retryHandlerFactory = new RetryThreeTimesWithBackoffFactory(kernel.Log); + Console.WriteLine("============================== RetryThreeTimesWithBackoff =============================="); + await RunRetryPolicyAsync(kernel, retryHandlerFactory); + + var retryHandlerFactory2 = new RetryThreeTimesWithRetryAfterBackoffFactory(kernel.Log); + Console.WriteLine("========================= RetryThreeTimesWithRetryAfterBackoff ========================="); + await RunRetryPolicyAsync(kernel, retryHandlerFactory2); + + Console.WriteLine("==================================== NoRetryPolicy ====================================="); + await RunRetryPolicyAsync(kernel, new NullHttpRetryHandlerFactory()); + + Console.WriteLine("=============================== DefaultHttpRetryHandler ================================"); + await RunRetryHandlerConfigAsync(new KernelConfig.HttpRetryConfig() { MaxRetryCount = 3, UseExponentialBackoff = true }); + + Console.WriteLine("======= DefaultHttpRetryConfig [MaxRetryCount = 3, UseExponentialBackoff = true] ====== "); + await RunRetryHandlerConfigAsync(new KernelConfig.HttpRetryConfig() { MaxRetryCount = 3, UseExponentialBackoff = true }); + } + + private static async Task RunRetryHandlerConfigAsync(KernelConfig.HttpRetryConfig? config = null) + { + var kernelBuilder = Kernel.Builder.WithLogger(ConsoleLogger.Log); + if (config != null) + { + kernelBuilder = kernelBuilder.Configure(c => c.SetDefaultHttpRetryConfig(config)); + } + + // Add 401 to the list of retryable status codes + // Typically 401 would not be something we retry but for demonstration + // purposes we are doing so as it's easy to trigger when using an invalid key. + kernelBuilder = kernelBuilder.Configure(c => c.DefaultHttpRetryConfig.RetryableStatusCodes.Add(System.Net.HttpStatusCode.Unauthorized)); + + // OpenAI settings - you can set the OPENAI_API_KEY to an invalid value to see the retry policy in play + kernelBuilder = kernelBuilder.Configure(c => c.AddOpenAICompletionBackend("text-davinci-003", "text-davinci-003", "BAD_KEY")); + + var kernel = kernelBuilder.Build(); + + await ImportAndExecuteSkillAsync(kernel); + } + + private static IKernel InitializeKernel() + { + var kernel = Kernel.Builder.WithLogger(ConsoleLogger.Log).Build(); + // OpenAI settings - you can set the OPENAI_API_KEY to an invalid value to see the retry policy in play + kernel.Config.AddOpenAICompletionBackend("text-davinci-003", "text-davinci-003", "BAD_KEY"); + + return kernel; + } + + private static async Task RunRetryPolicyAsync(IKernel kernel, IDelegatingHandlerFactory retryHandlerFactory) + { + kernel.Config.SetHttpHandlerFactory(retryHandlerFactory); + await ImportAndExecuteSkillAsync(kernel); + } + + private static async Task ImportAndExecuteSkillAsync(IKernel kernel) + { + // Load semantic skill defined with prompt templates + string folder = RepoFiles.SampleSkillsPath(); + + kernel.ImportSkill(new TimeSkill(), "time"); + + var qaSkill = kernel.ImportSemanticSkillFromDirectory( + folder, + "QASkill"); + + var question = "How popular is Polly library?"; + + // To see the retry policy in play, you can set the OPENAI_API_KEY to an invalid value + var answer = await kernel.RunAsync(question, qaSkill["Question"]); + + Console.WriteLine($"Question: {question}\n\n" + answer); + } +} + +/* Output: +============================== RetryThreeTimesWithBackoff ============================== +warn: object[0] + Error executing action [attempt 1 of 3], pausing 2000 msecs. Outcome: Unauthorized +warn: object[0] + Error executing action [attempt 2 of 3], pausing 4000 msecs. Outcome: Unauthorized +warn: object[0] + Error executing action [attempt 3 of 3], pausing 8000 msecs. Outcome: Unauthorized +fail: object[0] + Function call fail during pipeline step 0: QASkill.Question +Question: How popular is Polly library? + +Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized +========================= RetryThreeTimesWithRetryAfterBackoff ========================= +warn: object[0] + Error executing action [attempt 1 of 3], pausing 2000 msecs. Outcome: Unauthorized +warn: object[0] + Error executing action [attempt 2 of 3], pausing 2000 msecs. Outcome: Unauthorized +warn: object[0] + Error executing action [attempt 3 of 3], pausing 2000 msecs. Outcome: Unauthorized +Question: How popular is Polly library? + +Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized +fail: object[0] + Function call fail during pipeline step 0: QASkill.Question +=============================== DefaultHttpRetryHandler ================================ +warn: object[0] + Error executing action [attempt 1 of 1]. Reason: Unauthorized. Will retry after 2000ms +warn: object[0] + Error executing request, max retry count reached. Reason: Unauthorized +fail: object[0] + Function call fail during pipeline step 0: QASkill.Question +Question: How popular is Polly library? + +Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized +==================================== NoRetryPolicy ===================================== +Question: How popular is Polly library? + +Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized +fail: object[0] + Function call fail during pipeline step 0: QASkill.Question +======= DefaultHttpRetryConfig [MaxRetryCount = 3, UseExponentialBackoff = true] ====== +warn: object[0] + Error executing action [attempt 1 of 3]. Reason: Unauthorized. Will retry after 2000ms +warn: object[0] + Error executing action [attempt 2 of 3]. Reason: Unauthorized. Will retry after 4000ms +warn: object[0] + Error executing action [attempt 3 of 3]. Reason: Unauthorized. Will retry after 8000ms +warn: object[0] + Error executing request, max retry count reached. Reason: Unauthorized +fail: object[0] + Function call fail during pipeline step 0: QASkill.Question +Question: How popular is Polly library? + +Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized +== DONE == +*/ diff --git a/samples/dotnet/kernel-syntax-examples/Example08_RetryMechanism.cs b/samples/dotnet/kernel-syntax-examples/Example08_RetryMechanism.cs deleted file mode 100644 index 115d87b62e6a..000000000000 --- a/samples/dotnet/kernel-syntax-examples/Example08_RetryMechanism.cs +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.CoreSkills; -using Microsoft.SemanticKernel.KernelExtensions; -using Reliability; -using RepoUtils; - -// ReSharper disable once InconsistentNaming -public static class Example08_RetryMechanism -{ - public static async Task RunAsync() - { - Console.WriteLine("============================ RetryMechanism ============================"); - - IKernel kernel = Kernel.Builder.WithLogger(ConsoleLogger.Log).Build(); - kernel.Config.SetRetryMechanism(new RetryThreeTimesWithBackoff()); - - // OpenAI settings - kernel.Config.AddOpenAICompletionBackend("text-davinci-003", "text-davinci-003", Env.Var("OPENAI_API_KEY")); - - // Load semantic skill defined with prompt templates - string folder = RepoFiles.SampleSkillsPath(); - - kernel.ImportSkill(new TimeSkill(), "time"); - - var qaSkill = kernel.ImportSemanticSkillFromDirectory( - folder, - "QASkill"); - - var question = "How popular is Polly library?"; - var answer = await kernel.RunAsync(question, qaSkill["Question"]); - - Console.WriteLine($"Question: {question}\n\n" + answer); - } -} diff --git a/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs b/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs index f0038055c222..6bc930db92d3 100644 --- a/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs +++ b/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs @@ -9,7 +9,6 @@ using Microsoft.SemanticKernel.KernelExtensions; using Microsoft.SemanticKernel.Orchestration; using Microsoft.SemanticKernel.Orchestration.Extensions; -using Reliability; using RepoUtils; using Skills; @@ -46,7 +45,7 @@ private static async Task PoetrySamplesAsync() Console.WriteLine("Original plan:"); Console.WriteLine(originalPlan.Variables.ToPlan().PlanString); - _ = await ExecutePlanAsync(kernel, planner, originalPlan, 5); + await ExecutePlanAsync(kernel, planner, originalPlan, 5); } private static async Task EmailSamplesAsync() @@ -87,7 +86,7 @@ private static async Task EmailSamplesAsync() "and the kingdom was at peace once again. The king was so grateful to Mira that he asked her to marry him and she agreed. " + "They ruled the kingdom together, ruling with fairness and compassion, just as Arjun had done before. They lived " + "happily ever after, with the people of the kingdom remembering Mira as the brave young woman who saved them from the dragon."); - _ = await ExecutePlanAsync(kernel, planner, executionResults, 5); + await ExecutePlanAsync(kernel, planner, executionResults, 5); } private static async Task BookSamplesAsync() @@ -118,7 +117,7 @@ private static async Task BookSamplesAsync() Stopwatch sw = new(); sw.Start(); - _ = await ExecutePlanAsync(kernel, planner, originalPlan); + await ExecutePlanAsync(kernel, planner, originalPlan); } private static IKernel InitializeKernelAndPlanner(out IDictionary planner) @@ -129,7 +128,6 @@ private static IKernel InitializeKernelAndPlanner(out IDictionary + diff --git a/samples/dotnet/kernel-syntax-examples/Program.cs b/samples/dotnet/kernel-syntax-examples/Program.cs index c6ccf7aaf2ab..ef946a5c4fb4 100644 --- a/samples/dotnet/kernel-syntax-examples/Program.cs +++ b/samples/dotnet/kernel-syntax-examples/Program.cs @@ -30,7 +30,7 @@ public static async Task Main() await Example07_TemplateLanguage.RunAsync(); Console.WriteLine("== DONE =="); - await Example08_RetryMechanism.RunAsync(); + await Example08_RetryHandler.RunAsync(); Console.WriteLine("== DONE =="); await Example09_FunctionTypes.RunAsync(); diff --git a/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithBackoff.cs b/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithBackoff.cs index 157fe10c5acb..3e51a9fa9c32 100644 --- a/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithBackoff.cs +++ b/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithBackoff.cs @@ -1,38 +1,70 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.Reliability; using Polly; using Polly.Retry; namespace Reliability; +public class RetryThreeTimesWithBackoffFactory : IDelegatingHandlerFactory +{ + private readonly ILogger _log; + + public RetryThreeTimesWithBackoffFactory(ILogger log) + { + this._log = log; + } + + public DelegatingHandler Create() + { + return new RetryThreeTimesWithBackoff(this._log); + } +} + /// /// An example of a retry mechanism that retries three times with backoff. /// -public class RetryThreeTimesWithBackoff : IRetryMechanism +public class RetryThreeTimesWithBackoff : DelegatingHandler { - public Task ExecuteWithRetryAsync(Func action, ILogger log, CancellationToken cancellationToken = default) + private readonly AsyncRetryPolicy _policy; + + public RetryThreeTimesWithBackoff(ILogger log) + { + this._policy = GetPolicy(log); + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { - var policy = GetPolicy(log); - return policy.ExecuteAsync((_) => action(), cancellationToken); + return await this._policy.ExecuteAsync(async () => + { + var response = await base.SendAsync(request, cancellationToken); + return response; + }); } - private static AsyncRetryPolicy GetPolicy(ILogger log) + private static AsyncRetryPolicy GetPolicy(ILogger log) { + // Handle 429 and 401 errors + // Typically 401 would not be something we retry but for demonstration + // purposes we are doing so as it's easy to trigger when using an invalid key. return Policy - .Handle(ex => ex.ErrorCode == AIException.ErrorCodes.Throttling) + .HandleResult(response => + response.StatusCode is System.Net.HttpStatusCode.TooManyRequests or System.Net.HttpStatusCode.Unauthorized) .WaitAndRetryAsync(new[] { TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(4), TimeSpan.FromSeconds(8) }, - (ex, timespan, retryCount, _) => log.LogWarning(ex, - "Error executing action [attempt {0} of ], pausing {1} msecs", retryCount, timespan.TotalMilliseconds)); + (outcome, timespan, retryCount, _) => log.LogWarning( + "Error executing action [attempt {0} of 3], pausing {1} msecs. Outcome: {2}", + retryCount, + timespan.TotalMilliseconds, + outcome.Result.StatusCode)); } } diff --git a/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithRetryAfterBackoff.cs b/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithRetryAfterBackoff.cs new file mode 100644 index 000000000000..d04cd8e0302b --- /dev/null +++ b/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithRetryAfterBackoff.cs @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Reliability; +using Polly; +using Polly.Retry; + +namespace Reliability; + +public class RetryThreeTimesWithRetryAfterBackoffFactory : IDelegatingHandlerFactory +{ + private readonly ILogger _log; + + public RetryThreeTimesWithRetryAfterBackoffFactory(ILogger log) + { + this._log = log; + } + + public DelegatingHandler Create() + { + return new RetryThreeTimesWithRetryAfterBackoff(this._log); + } +} + +/// +/// An example of a retry mechanism that retries three times with backoff using the RetryAfter value. +/// +public class RetryThreeTimesWithRetryAfterBackoff : DelegatingHandler +{ + private readonly AsyncRetryPolicy _policy; + + public RetryThreeTimesWithRetryAfterBackoff(ILogger log) + { + this._policy = GetPolicy(log); + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + return await this._policy.ExecuteAsync(async () => + { + var response = await base.SendAsync(request, cancellationToken); + return response; + }); + } + + private static AsyncRetryPolicy GetPolicy(ILogger log) + { + // Handle 429 and 401 errors + // Typically 401 would not be something we retry but for demonstration + // purposes we are doing so as it's easy to trigger when using an invalid key. + return Policy + .HandleResult(response => + response.StatusCode is System.Net.HttpStatusCode.TooManyRequests or System.Net.HttpStatusCode.Unauthorized) + .WaitAndRetryAsync( + retryCount: 3, + sleepDurationProvider: (_, r, _) => + { + var response = r.Result; + var retryAfter = response.Headers.RetryAfter?.Delta ?? response.Headers.RetryAfter?.Date - DateTimeOffset.Now; + return retryAfter ?? TimeSpan.FromSeconds(2); + }, + (outcome, timespan, retryCount, _) => + { + log.LogWarning( + "Error executing action [attempt {0} of 3], pausing {1} msecs. Outcome: {2}", + retryCount, + timespan.TotalMilliseconds, + outcome.Result.StatusCode); + return Task.CompletedTask; + }); + } +} From 79d8ac12327ed10729df7d134d92e91fe0cecd27 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Fri, 10 Mar 2023 12:30:03 -0800 Subject: [PATCH 2/8] Downgrade nuget package version to 0.8 Summary: This commit changes the version prefix of all nuget packages from 0.9 to 0.8 in the nuget-package.props file. This is done to align with the current release cycle and avoid confusion with the previous versions. --- dotnet/nuget/nuget-package.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/nuget/nuget-package.props b/dotnet/nuget/nuget-package.props index 24e4f41f93fe..9f95907c4486 100644 --- a/dotnet/nuget/nuget-package.props +++ b/dotnet/nuget/nuget-package.props @@ -3,7 +3,7 @@ true true - 0.9 + 0.8 From 2168f692843f16d13ae3d4b42c6d4deb18e43a27 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Fri, 10 Mar 2023 12:30:39 -0800 Subject: [PATCH 3/8] Refactor HTTP retry handler logic and logging Summary: This commit makes several changes to the HTTP retry handler logic and logging for the Semantic Kernel project. The main changes are: - Pass the logger instance to the retry handler factory instead of the constructor, to allow different loggers for different clients. - Change the log level from warning to error when the max retry count or time is reached, to indicate a more severe problem. - Remove the TimeoutException and WebException from the list of retryable exception types, since they are already handled by the HttpRequestException. - Rename the WithRetryHandler method to WithRetryHandlerFactory, to avoid confusion with the DelegatingHandler class. - Fix a typo in the CloneAsync method comment. --- .../AI/OpenAI/Clients/OpenAIClientAbstract.cs | 3 +-- dotnet/src/SemanticKernel/Configuration/KernelConfig.cs | 4 +--- dotnet/src/SemanticKernel/KernelBuilder.cs | 4 ++-- .../SemanticKernel/Reliability/DefaultHttpRetryHandler.cs | 6 +++--- .../Reliability/DefaultHttpRetryHandlerFactory.cs | 8 +++----- .../Reliability/IDelegatingHandlerFactory.cs | 3 ++- .../SemanticKernel/Reliability/NullHttpRetryHandler.cs | 3 ++- 7 files changed, 14 insertions(+), 17 deletions(-) diff --git a/dotnet/src/SemanticKernel/AI/OpenAI/Clients/OpenAIClientAbstract.cs b/dotnet/src/SemanticKernel/AI/OpenAI/Clients/OpenAIClientAbstract.cs index 874d0ed59603..022f35ce15f0 100644 --- a/dotnet/src/SemanticKernel/AI/OpenAI/Clients/OpenAIClientAbstract.cs +++ b/dotnet/src/SemanticKernel/AI/OpenAI/Clients/OpenAIClientAbstract.cs @@ -44,7 +44,7 @@ internal OpenAIClientAbstract(ILogger? log = null, IDelegatingHandlerFactory? ha this._handlerFactory = handlerFactory ?? this._handlerFactory; this._httpClientHandler = new() { CheckCertificateRevocationList = true }; - this._retryHandler = this._handlerFactory.Create(); + this._retryHandler = this._handlerFactory.Create(this.Log); this._retryHandler.InnerHandler = this._httpClientHandler; this.HTTPClient = new HttpClient(this._retryHandler); @@ -148,7 +148,6 @@ private async Task ExecutePostRequestAsync(string url, string requestBody, try { using HttpContent content = new StringContent(requestBody, Encoding.UTF8, MediaTypeNames.Application.Json); - HttpResponseMessage response = await this.HTTPClient.PostAsync(url, content, cancellationToken); if (response == null) diff --git a/dotnet/src/SemanticKernel/Configuration/KernelConfig.cs b/dotnet/src/SemanticKernel/Configuration/KernelConfig.cs index 2617b621b27f..63739d2a969d 100644 --- a/dotnet/src/SemanticKernel/Configuration/KernelConfig.cs +++ b/dotnet/src/SemanticKernel/Configuration/KernelConfig.cs @@ -76,8 +76,6 @@ public int MaxRetryCount /// public List RetryableExceptionTypes { get; set; } = new() { - typeof(TimeoutException), - typeof(WebException), typeof(HttpRequestException) }; @@ -87,7 +85,7 @@ public int MaxRetryCount /// /// Global retry logic used for all the backends http calls /// - public IDelegatingHandlerFactory HttpHandlerFactory { get; private set; } = new DefaultHttpRetryHandlerFactory(new HttpRetryConfig(), NullLogger.Instance); + public IDelegatingHandlerFactory HttpHandlerFactory { get; private set; } = new DefaultHttpRetryHandlerFactory(new HttpRetryConfig()); public HttpRetryConfig DefaultHttpRetryConfig { get; private set; } = new(); diff --git a/dotnet/src/SemanticKernel/KernelBuilder.cs b/dotnet/src/SemanticKernel/KernelBuilder.cs index 14f724c0c546..4da19625be8c 100644 --- a/dotnet/src/SemanticKernel/KernelBuilder.cs +++ b/dotnet/src/SemanticKernel/KernelBuilder.cs @@ -43,7 +43,7 @@ public static IKernel Create() /// Kernel instance public IKernel Build() { - this._config.SetHttpHandlerFactory(this._httpHandlerFactory ?? new DefaultHttpRetryHandlerFactory(this._config.DefaultHttpRetryConfig, this._log)); + this._config.SetHttpHandlerFactory(this._httpHandlerFactory ?? new DefaultHttpRetryHandlerFactory(this._config.DefaultHttpRetryConfig)); var instance = new Kernel( new SkillCollection(this._log), @@ -113,7 +113,7 @@ public KernelBuilder WithMemoryStorageAndEmbeddingGenerator( return this; } - public KernelBuilder WithRetryHandler(IDelegatingHandlerFactory httpHandlerFactory) + public KernelBuilder WithRetryHandlerFactory(IDelegatingHandlerFactory httpHandlerFactory) { Verify.NotNull(httpHandlerFactory, "The retry handler factory instance provided is NULL"); this._httpHandlerFactory = httpHandlerFactory; diff --git a/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandler.cs b/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandler.cs index 1e29e93a1d7e..c0bf8bedb885 100644 --- a/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandler.cs +++ b/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandler.cs @@ -85,7 +85,7 @@ protected override async Task SendAsync(HttpRequestMessage if (retryCount >= this._config.MaxRetryCount) { - this._log.LogWarning( + this._log.LogError( "Error executing request, max retry count reached. Reason: {0}", reason); return response; } @@ -94,7 +94,7 @@ protected override async Task SendAsync(HttpRequestMessage // just return if (!this.HasTimeForRetry(start, retryCount, response, out waitFor)) { - this._log.LogWarning( + this._log.LogError( "Error executing request, max total retry time reached. Reason: {0}", reason); return response; } @@ -178,7 +178,7 @@ private bool ShouldRetry(Exception exception) /// The previous needs to be copy. /// The . /// - /// Re-issue a new HTTP request with the previous request's headers and properities + /// Re-issue a new HTTP request with the previous request's headers and properties /// internal static async Task CloneAsync(HttpRequestMessage originalRequest) { diff --git a/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandlerFactory.cs b/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandlerFactory.cs index e7673261d4f8..5651bed728cd 100644 --- a/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandlerFactory.cs +++ b/dotnet/src/SemanticKernel/Reliability/DefaultHttpRetryHandlerFactory.cs @@ -8,17 +8,15 @@ namespace Microsoft.SemanticKernel.Reliability; internal class DefaultHttpRetryHandlerFactory : IDelegatingHandlerFactory { - internal DefaultHttpRetryHandlerFactory(KernelConfig.HttpRetryConfig? config = null, ILogger? log = null) + internal DefaultHttpRetryHandlerFactory(KernelConfig.HttpRetryConfig? config = null) { this._config = config; - this._log = log; } - public DelegatingHandler Create() + public DelegatingHandler Create(ILogger log) { - return new DefaultHttpRetryHandler(this._config, this._log); + return new DefaultHttpRetryHandler(this._config, log); } private readonly KernelConfig.HttpRetryConfig? _config; - private readonly ILogger? _log; } diff --git a/dotnet/src/SemanticKernel/Reliability/IDelegatingHandlerFactory.cs b/dotnet/src/SemanticKernel/Reliability/IDelegatingHandlerFactory.cs index fdf9b1123745..1d4d4d29f8ae 100644 --- a/dotnet/src/SemanticKernel/Reliability/IDelegatingHandlerFactory.cs +++ b/dotnet/src/SemanticKernel/Reliability/IDelegatingHandlerFactory.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using System.Net.Http; +using Microsoft.Extensions.Logging; namespace Microsoft.SemanticKernel.Reliability; @@ -9,5 +10,5 @@ namespace Microsoft.SemanticKernel.Reliability; /// public interface IDelegatingHandlerFactory { - DelegatingHandler Create(); + DelegatingHandler Create(ILogger log); } diff --git a/dotnet/src/SemanticKernel/Reliability/NullHttpRetryHandler.cs b/dotnet/src/SemanticKernel/Reliability/NullHttpRetryHandler.cs index b8f5988b1237..8749ace9ef51 100644 --- a/dotnet/src/SemanticKernel/Reliability/NullHttpRetryHandler.cs +++ b/dotnet/src/SemanticKernel/Reliability/NullHttpRetryHandler.cs @@ -1,12 +1,13 @@ // Copyright (c) Microsoft. All rights reserved. using System.Net.Http; +using Microsoft.Extensions.Logging; namespace Microsoft.SemanticKernel.Reliability; public class NullHttpRetryHandlerFactory : IDelegatingHandlerFactory { - public DelegatingHandler Create() + public DelegatingHandler Create(ILogger log) { return new NullHttpRetryHandler(); } From 515d7bb37005f39c913e6cf6c42806699a3c8bd9 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Fri, 10 Mar 2023 12:33:07 -0800 Subject: [PATCH 4/8] This commit refactors and improves the logging for the retry handlers in the RepoUtils project. It makes the following changes: - Simplifies the creation of retry handler factories by passing the logger as a parameter instead of a constructor dependency. This avoids creating multiple logger instances for each factory. - Removes the unused TimeoutException and WebException types from the retry handler tests, since they are no longer used by the HttpClient. - Changes the logging format to use milliseconds instead of msecs for consistency and clarity. It also fixes the formatting of the time span values. - Adds some missing log messages for the NoRetryPolicy and the DefaultHttpRetryHandler. - Lowers the minimum level for the console logger to show all messages, and adds a filter for the System namespace to only show warnings or higher. This makes the console output more informative and less noisy. --- .../DefaultHttpRetryHandlerTests.cs | 8 -- samples/dotnet/KernelBuilder/Program.cs | 6 +- .../Example08_RetryHandler.cs | 109 +++++++++++------- .../Reliability/RetryThreeTimesWithBackoff.cs | 13 +-- .../RetryThreeTimesWithRetryAfterBackoff.cs | 13 +-- .../RepoUtils/ConsoleLogger.cs | 6 +- 6 files changed, 80 insertions(+), 75 deletions(-) diff --git a/dotnet/src/SemanticKernel.Test/Reliability/DefaultHttpRetryHandlerTests.cs b/dotnet/src/SemanticKernel.Test/Reliability/DefaultHttpRetryHandlerTests.cs index 254e1b99ef30..5a051dccbcd1 100644 --- a/dotnet/src/SemanticKernel.Test/Reliability/DefaultHttpRetryHandlerTests.cs +++ b/dotnet/src/SemanticKernel.Test/Reliability/DefaultHttpRetryHandlerTests.cs @@ -69,8 +69,6 @@ public async Task ItRetriesOnceOnRetryableStatusAsync(HttpStatusCode statusCode) } [Theory] - [InlineData(typeof(TimeoutException))] - [InlineData(typeof(WebException))] [InlineData(typeof(HttpRequestException))] public async Task ItRetriesOnceOnRetryableExceptionAsync(Type exceptionType) { @@ -92,8 +90,6 @@ public async Task ItRetriesOnceOnRetryableExceptionAsync(Type exceptionType) } [Theory] - [InlineData(typeof(TimeoutException))] - [InlineData(typeof(WebException))] [InlineData(typeof(HttpRequestException))] public async Task NoMaxRetryCountCallsOnceForExceptionAsync(Type exceptionType) { @@ -140,8 +136,6 @@ public async Task ItRetriesOnceOnTransientStatusWithExponentialBackoffAsync(Http } [Theory] - [InlineData(typeof(TimeoutException))] - [InlineData(typeof(WebException))] [InlineData(typeof(HttpRequestException))] public async Task ItRetriesOnceOnRetryableExceptionWithExponentialBackoffAsync(Type exceptionType) { @@ -257,8 +251,6 @@ public async Task ItRetriesStatusCustomCountAsync(HttpStatusCode expectedStatus) } [Theory] - [InlineData(typeof(TimeoutException))] - [InlineData(typeof(WebException))] [InlineData(typeof(HttpRequestException))] public async Task ItRetriesExceptionsCustomCountAsync(Type expectedException) { diff --git a/samples/dotnet/KernelBuilder/Program.cs b/samples/dotnet/KernelBuilder/Program.cs index 764381fe66af..b7653a21a0d7 100644 --- a/samples/dotnet/KernelBuilder/Program.cs +++ b/samples/dotnet/KernelBuilder/Program.cs @@ -109,9 +109,9 @@ public class RetryThreeTimesFactory : IDelegatingHandlerFactory { - public DelegatingHandler Create() + public DelegatingHandler Create(ILogger log) { - return new RetryThreeTimes(); + return new RetryThreeTimes(log); } } @@ -144,7 +144,7 @@ private static AsyncRetryPolicy GetPolicy(ILogger log) TimeSpan.FromSeconds(8) }, (ex, timespan, retryCount, _) => log.LogWarning(ex, - "Error executing action [attempt {0} of 3], pausing {1} msecs", + "Error executing action [attempt {0} of 3], pausing {1}ms", retryCount, timespan.TotalMilliseconds)); } } diff --git a/samples/dotnet/kernel-syntax-examples/Example08_RetryHandler.cs b/samples/dotnet/kernel-syntax-examples/Example08_RetryHandler.cs index fcd64a20c8fe..0f0e1fad33ce 100644 --- a/samples/dotnet/kernel-syntax-examples/Example08_RetryHandler.cs +++ b/samples/dotnet/kernel-syntax-examples/Example08_RetryHandler.cs @@ -2,6 +2,7 @@ using System; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.Configuration; using Microsoft.SemanticKernel.CoreSkills; @@ -16,21 +17,20 @@ public static class Example08_RetryHandler public static async Task RunAsync() { var kernel = InitializeKernel(); - var retryHandlerFactory = new RetryThreeTimesWithBackoffFactory(kernel.Log); - Console.WriteLine("============================== RetryThreeTimesWithBackoff =============================="); + var retryHandlerFactory = new RetryThreeTimesWithBackoffFactory(); + ConsoleLogger.Log.LogInformation("============================== RetryThreeTimesWithBackoff =============================="); await RunRetryPolicyAsync(kernel, retryHandlerFactory); - var retryHandlerFactory2 = new RetryThreeTimesWithRetryAfterBackoffFactory(kernel.Log); - Console.WriteLine("========================= RetryThreeTimesWithRetryAfterBackoff ========================="); - await RunRetryPolicyAsync(kernel, retryHandlerFactory2); + ConsoleLogger.Log.LogInformation("========================= RetryThreeTimesWithRetryAfterBackoff ========================="); + await RunRetryPolicyBuilderAsync(typeof(RetryThreeTimesWithRetryAfterBackoffFactory)); - Console.WriteLine("==================================== NoRetryPolicy ====================================="); - await RunRetryPolicyAsync(kernel, new NullHttpRetryHandlerFactory()); + ConsoleLogger.Log.LogInformation("==================================== NoRetryPolicy ====================================="); + await RunRetryPolicyBuilderAsync(typeof(NullHttpRetryHandlerFactory)); - Console.WriteLine("=============================== DefaultHttpRetryHandler ================================"); + ConsoleLogger.Log.LogInformation("=============================== DefaultHttpRetryHandler ================================"); await RunRetryHandlerConfigAsync(new KernelConfig.HttpRetryConfig() { MaxRetryCount = 3, UseExponentialBackoff = true }); - Console.WriteLine("======= DefaultHttpRetryConfig [MaxRetryCount = 3, UseExponentialBackoff = true] ====== "); + ConsoleLogger.Log.LogInformation("======= DefaultHttpRetryConfig [MaxRetryCount = 3, UseExponentialBackoff = true] ======="); await RunRetryHandlerConfigAsync(new KernelConfig.HttpRetryConfig() { MaxRetryCount = 3, UseExponentialBackoff = true }); } @@ -70,6 +70,18 @@ private static async Task RunRetryPolicyAsync(IKernel kernel, IDelegatingHandler await ImportAndExecuteSkillAsync(kernel); } + private static async Task RunRetryPolicyBuilderAsync(Type retryHandlerFactoryType) + { + var kernelBuilder = Kernel.Builder.WithLogger(ConsoleLogger.Log).WithRetryHandlerFactory((Activator.CreateInstance(retryHandlerFactoryType) as IDelegatingHandlerFactory)!); + + // OpenAI settings - you can set the OPENAI_API_KEY to an invalid value to see the retry policy in play + kernelBuilder = kernelBuilder.Configure(c => c.AddOpenAICompletionBackend("text-davinci-003", "text-davinci-003", "BAD_KEY")); + + var kernel = kernelBuilder.Build(); + + await ImportAndExecuteSkillAsync(kernel); + } + private static async Task ImportAndExecuteSkillAsync(IKernel kernel) { // Load semantic skill defined with prompt templates @@ -83,67 +95,80 @@ private static async Task ImportAndExecuteSkillAsync(IKernel kernel) var question = "How popular is Polly library?"; + ConsoleLogger.Log.LogInformation("Question: {0}", question); // To see the retry policy in play, you can set the OPENAI_API_KEY to an invalid value var answer = await kernel.RunAsync(question, qaSkill["Question"]); - - Console.WriteLine($"Question: {question}\n\n" + answer); + ConsoleLogger.Log.LogInformation("Answer: {0}", answer); } } /* Output: -============================== RetryThreeTimesWithBackoff ============================== +info: object[0] + ============================== RetryThreeTimesWithBackoff ============================== +info: object[0] + Question: How popular is Polly library? warn: object[0] - Error executing action [attempt 1 of 3], pausing 2000 msecs. Outcome: Unauthorized + Error executing action [attempt 1 of 3], pausing 2000ms. Outcome: Unauthorized warn: object[0] - Error executing action [attempt 2 of 3], pausing 4000 msecs. Outcome: Unauthorized + Error executing action [attempt 2 of 3], pausing 4000ms. Outcome: Unauthorized warn: object[0] - Error executing action [attempt 3 of 3], pausing 8000 msecs. Outcome: Unauthorized + Error executing action [attempt 3 of 3], pausing 8000ms. Outcome: Unauthorized fail: object[0] Function call fail during pipeline step 0: QASkill.Question -Question: How popular is Polly library? - -Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized -========================= RetryThreeTimesWithRetryAfterBackoff ========================= +info: object[0] + Answer: Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized +info: object[0] + ========================= RetryThreeTimesWithRetryAfterBackoff ========================= +info: object[0] + Question: How popular is Polly library? warn: object[0] - Error executing action [attempt 1 of 3], pausing 2000 msecs. Outcome: Unauthorized + Error executing action [attempt 1 of 3], pausing 2000ms. Outcome: Unauthorized warn: object[0] - Error executing action [attempt 2 of 3], pausing 2000 msecs. Outcome: Unauthorized + Error executing action [attempt 2 of 3], pausing 2000ms. Outcome: Unauthorized warn: object[0] - Error executing action [attempt 3 of 3], pausing 2000 msecs. Outcome: Unauthorized -Question: How popular is Polly library? - -Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized + Error executing action [attempt 3 of 3], pausing 2000ms. Outcome: Unauthorized fail: object[0] Function call fail during pipeline step 0: QASkill.Question -=============================== DefaultHttpRetryHandler ================================ +info: object[0] + Answer: Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized +info: object[0] + ==================================== NoRetryPolicy ===================================== +info: object[0] + Question: How popular is Polly library? +fail: object[0] + Function call fail during pipeline step 0: QASkill.Question +info: object[0] + Answer: Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized +info: object[0] + =============================== DefaultHttpRetryHandler ================================ +info: object[0] + Question: How popular is Polly library? warn: object[0] - Error executing action [attempt 1 of 1]. Reason: Unauthorized. Will retry after 2000ms + Error executing action [attempt 1 of 3]. Reason: Unauthorized. Will retry after 2000ms warn: object[0] - Error executing request, max retry count reached. Reason: Unauthorized + Error executing action [attempt 2 of 3]. Reason: Unauthorized. Will retry after 4000ms +warn: object[0] + Error executing action [attempt 3 of 3]. Reason: Unauthorized. Will retry after 8000ms fail: object[0] - Function call fail during pipeline step 0: QASkill.Question -Question: How popular is Polly library? - -Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized -==================================== NoRetryPolicy ===================================== -Question: How popular is Polly library? - -Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized + Error executing request, max retry count reached. Reason: Unauthorized fail: object[0] Function call fail during pipeline step 0: QASkill.Question -======= DefaultHttpRetryConfig [MaxRetryCount = 3, UseExponentialBackoff = true] ====== +info: object[0] + Answer: Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized +info: object[0] + ======= DefaultHttpRetryConfig [MaxRetryCount = 3, UseExponentialBackoff = true] ======= +info: object[0] + Question: How popular is Polly library? warn: object[0] Error executing action [attempt 1 of 3]. Reason: Unauthorized. Will retry after 2000ms warn: object[0] Error executing action [attempt 2 of 3]. Reason: Unauthorized. Will retry after 4000ms warn: object[0] Error executing action [attempt 3 of 3]. Reason: Unauthorized. Will retry after 8000ms -warn: object[0] +fail: object[0] Error executing request, max retry count reached. Reason: Unauthorized fail: object[0] Function call fail during pipeline step 0: QASkill.Question -Question: How popular is Polly library? - -Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized -== DONE == +info: object[0] + Answer: Error: AccessDenied: The request is not authorized, HTTP status: Unauthorized */ diff --git a/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithBackoff.cs b/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithBackoff.cs index 3e51a9fa9c32..bc7e644b696a 100644 --- a/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithBackoff.cs +++ b/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithBackoff.cs @@ -13,16 +13,9 @@ namespace Reliability; public class RetryThreeTimesWithBackoffFactory : IDelegatingHandlerFactory { - private readonly ILogger _log; - - public RetryThreeTimesWithBackoffFactory(ILogger log) - { - this._log = log; - } - - public DelegatingHandler Create() + public DelegatingHandler Create(ILogger log) { - return new RetryThreeTimesWithBackoff(this._log); + return new RetryThreeTimesWithBackoff(log); } } @@ -62,7 +55,7 @@ private static AsyncRetryPolicy GetPolicy(ILogger log) TimeSpan.FromSeconds(8) }, (outcome, timespan, retryCount, _) => log.LogWarning( - "Error executing action [attempt {0} of 3], pausing {1} msecs. Outcome: {2}", + "Error executing action [attempt {0} of 3], pausing {1}ms. Outcome: {2}", retryCount, timespan.TotalMilliseconds, outcome.Result.StatusCode)); diff --git a/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithRetryAfterBackoff.cs b/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithRetryAfterBackoff.cs index d04cd8e0302b..2da172becf75 100644 --- a/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithRetryAfterBackoff.cs +++ b/samples/dotnet/kernel-syntax-examples/Reliability/RetryThreeTimesWithRetryAfterBackoff.cs @@ -13,16 +13,9 @@ namespace Reliability; public class RetryThreeTimesWithRetryAfterBackoffFactory : IDelegatingHandlerFactory { - private readonly ILogger _log; - - public RetryThreeTimesWithRetryAfterBackoffFactory(ILogger log) - { - this._log = log; - } - - public DelegatingHandler Create() + public DelegatingHandler Create(ILogger log) { - return new RetryThreeTimesWithRetryAfterBackoff(this._log); + return new RetryThreeTimesWithRetryAfterBackoff(log); } } @@ -66,7 +59,7 @@ private static AsyncRetryPolicy GetPolicy(ILogger log) (outcome, timespan, retryCount, _) => { log.LogWarning( - "Error executing action [attempt {0} of 3], pausing {1} msecs. Outcome: {2}", + "Error executing action [attempt {0} of 3], pausing {1}ms. Outcome: {2}", retryCount, timespan.TotalMilliseconds, outcome.Result.StatusCode); diff --git a/samples/dotnet/kernel-syntax-examples/RepoUtils/ConsoleLogger.cs b/samples/dotnet/kernel-syntax-examples/RepoUtils/ConsoleLogger.cs index 51aa7055668b..a3d7ff63a8f9 100644 --- a/samples/dotnet/kernel-syntax-examples/RepoUtils/ConsoleLogger.cs +++ b/samples/dotnet/kernel-syntax-examples/RepoUtils/ConsoleLogger.cs @@ -13,18 +13,20 @@ internal static class ConsoleLogger internal static ILogger Log => LogFactory.CreateLogger(); private static ILoggerFactory LogFactory => s_loggerFactory.Value; + private static readonly Lazy s_loggerFactory = new(LogBuilder); private static ILoggerFactory LogBuilder() { return LoggerFactory.Create(builder => { - builder.SetMinimumLevel(LogLevel.Warning); + // builder.SetMinimumLevel(LogLevel.Warning); // builder.AddFilter("Microsoft", LogLevel.Trace); // builder.AddFilter("Microsoft", LogLevel.Debug); // builder.AddFilter("Microsoft", LogLevel.Information); - // builder.AddFilter("Microsoft", LogLevel.Warning); + builder.AddFilter("Microsoft", LogLevel.Warning); // builder.AddFilter("Microsoft", LogLevel.Error); + builder.AddFilter("System", LogLevel.Warning); builder.AddConsole(); }); } From e15cae2ea732c1aca0f0ae65ccecc718c77079e0 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Fri, 10 Mar 2023 12:47:04 -0800 Subject: [PATCH 5/8] formatting --- .../dotnet/kernel-syntax-examples/Example08_RetryHandler.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/dotnet/kernel-syntax-examples/Example08_RetryHandler.cs b/samples/dotnet/kernel-syntax-examples/Example08_RetryHandler.cs index 0f0e1fad33ce..812531bd80f2 100644 --- a/samples/dotnet/kernel-syntax-examples/Example08_RetryHandler.cs +++ b/samples/dotnet/kernel-syntax-examples/Example08_RetryHandler.cs @@ -72,7 +72,8 @@ private static async Task RunRetryPolicyAsync(IKernel kernel, IDelegatingHandler private static async Task RunRetryPolicyBuilderAsync(Type retryHandlerFactoryType) { - var kernelBuilder = Kernel.Builder.WithLogger(ConsoleLogger.Log).WithRetryHandlerFactory((Activator.CreateInstance(retryHandlerFactoryType) as IDelegatingHandlerFactory)!); + var kernelBuilder = Kernel.Builder.WithLogger(ConsoleLogger.Log) + .WithRetryHandlerFactory((Activator.CreateInstance(retryHandlerFactoryType) as IDelegatingHandlerFactory)!); // OpenAI settings - you can set the OPENAI_API_KEY to an invalid value to see the retry policy in play kernelBuilder = kernelBuilder.Configure(c => c.AddOpenAICompletionBackend("text-davinci-003", "text-davinci-003", "BAD_KEY")); From 6303bb6638778cb9e49427e1e667c7015b9d7391 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Fri, 10 Mar 2023 13:14:47 -0800 Subject: [PATCH 6/8] undo formatting not related --- dotnet/src/SemanticKernel.Test/KernelTests.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dotnet/src/SemanticKernel.Test/KernelTests.cs b/dotnet/src/SemanticKernel.Test/KernelTests.cs index 253f20514f17..210d3df0e859 100644 --- a/dotnet/src/SemanticKernel.Test/KernelTests.cs +++ b/dotnet/src/SemanticKernel.Test/KernelTests.cs @@ -139,8 +139,7 @@ public void ItFailsIfCompletionBackendConfigIsNotSet() // Arrange var kernel = KernelBuilder.Create(); - var exception = Assert.Throws(() => - kernel.CreateSemanticFunction(promptTemplate: "Tell me a joke", functionName: "joker", skillName: "jk", description: "Nice fun")); + var exception = Assert.Throws(() => kernel.CreateSemanticFunction(promptTemplate: "Tell me a joke", functionName: "joker", skillName: "jk", description: "Nice fun")); } public class MySkill From d21502c1c13656f45a5d06da6c1897e0962f9cc5 Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Fri, 10 Mar 2023 13:24:22 -0800 Subject: [PATCH 7/8] renaming --- .../Configuration/KernelConfigTests.cs | 14 +++++++------- .../SemanticKernel/Configuration/KernelConfig.cs | 4 ++-- dotnet/src/SemanticKernel/KernelBuilder.cs | 2 +- samples/dotnet/KernelBuilder/Program.cs | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/dotnet/src/SemanticKernel.Test/Configuration/KernelConfigTests.cs b/dotnet/src/SemanticKernel.Test/Configuration/KernelConfigTests.cs index e3a620e48314..ee1c3883147a 100644 --- a/dotnet/src/SemanticKernel.Test/Configuration/KernelConfigTests.cs +++ b/dotnet/src/SemanticKernel.Test/Configuration/KernelConfigTests.cs @@ -17,48 +17,48 @@ namespace SemanticKernelTests.Configuration; public class KernelConfigTests { [Fact] - public void HttpRetryPolicyIsSet() + public void HttpRetryHandlerFactoryIsSet() { // Arrange var retry = new NullHttpRetryHandlerFactory(); var config = new KernelConfig(); // Act - config.SetHttpHandlerFactory(retry); + config.SetHttpRetryHandlerFactory(retry); // Assert Assert.Equal(retry, config.HttpHandlerFactory); } [Fact] - public void HttpRetryPolicyIsSetWithCustomImplementation() + public void HttpRetryHandlerFactoryIsSetWithCustomImplementation() { // Arrange var retry = new Mock(); var config = new KernelConfig(); // Act - config.SetHttpHandlerFactory(retry.Object); + config.SetHttpRetryHandlerFactory(retry.Object); // Assert Assert.Equal(retry.Object, config.HttpHandlerFactory); } [Fact] - public void HttpRetryPolicyIsSetToDefaultHttpHandlerFactoryIfNull() + public void HttpRetryHandlerFactoryIsSetToDefaultHttpRetryHandlerFactoryIfNull() { // Arrange var config = new KernelConfig(); // Act - config.SetHttpHandlerFactory(null); + config.SetHttpRetryHandlerFactory(null); // Assert Assert.IsType(config.HttpHandlerFactory); } [Fact] - public void HttpRetryPolicyIsSetToDefaultHttpHandlerFactoryIfNotSet() + public void HttpRetryHandlerFactoryIsSetToDefaultHttpRetryHandlerFactoryIfNotSet() { // Arrange var config = new KernelConfig(); diff --git a/dotnet/src/SemanticKernel/Configuration/KernelConfig.cs b/dotnet/src/SemanticKernel/Configuration/KernelConfig.cs index 63739d2a969d..132b74de66c5 100644 --- a/dotnet/src/SemanticKernel/Configuration/KernelConfig.cs +++ b/dotnet/src/SemanticKernel/Configuration/KernelConfig.cs @@ -252,9 +252,9 @@ public bool HasEmbeddingsBackend(string label, Func? condi /// /// Set the http retry handler factory to use for the kernel. /// - /// Retry handler factory to use. + /// Http retry handler factory to use. /// The updated kernel configuration. - public KernelConfig SetHttpHandlerFactory(IDelegatingHandlerFactory? httpHandlerFactory = null) + public KernelConfig SetHttpRetryHandlerFactory(IDelegatingHandlerFactory? httpHandlerFactory = null) { if (httpHandlerFactory != null) { diff --git a/dotnet/src/SemanticKernel/KernelBuilder.cs b/dotnet/src/SemanticKernel/KernelBuilder.cs index 4da19625be8c..6c5a79fd6c33 100644 --- a/dotnet/src/SemanticKernel/KernelBuilder.cs +++ b/dotnet/src/SemanticKernel/KernelBuilder.cs @@ -43,7 +43,7 @@ public static IKernel Create() /// Kernel instance public IKernel Build() { - this._config.SetHttpHandlerFactory(this._httpHandlerFactory ?? new DefaultHttpRetryHandlerFactory(this._config.DefaultHttpRetryConfig)); + this._config.SetHttpRetryHandlerFactory(this._httpHandlerFactory ?? new DefaultHttpRetryHandlerFactory(this._config.DefaultHttpRetryConfig)); var instance = new Kernel( new SkillCollection(this._log), diff --git a/samples/dotnet/KernelBuilder/Program.cs b/samples/dotnet/KernelBuilder/Program.cs index b7653a21a0d7..2fef62390d24 100644 --- a/samples/dotnet/KernelBuilder/Program.cs +++ b/samples/dotnet/KernelBuilder/Program.cs @@ -104,7 +104,7 @@ // AI requests (when using the kernel). var kernel8 = Kernel.Builder - .Configure(c => c.SetHttpHandlerFactory(new RetryThreeTimesFactory())) + .Configure(c => c.SetHttpRetryHandlerFactory(new RetryThreeTimesFactory())) .Build(); public class RetryThreeTimesFactory : IDelegatingHandlerFactory From 7d43d636c1374e797bfe0048ce86b66c664bc4ce Mon Sep 17 00:00:00 2001 From: "lemiller@microsoft.com" Date: Fri, 10 Mar 2023 13:28:20 -0800 Subject: [PATCH 8/8] formatting --- .../Configuration/KernelConfigTests.cs | 16 ---------------- .../OpenAI/Clients/AzureOpenAIClientAbstract.cs | 5 ++--- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/dotnet/src/SemanticKernel.Test/Configuration/KernelConfigTests.cs b/dotnet/src/SemanticKernel.Test/Configuration/KernelConfigTests.cs index ee1c3883147a..3134586ab43d 100644 --- a/dotnet/src/SemanticKernel.Test/Configuration/KernelConfigTests.cs +++ b/dotnet/src/SemanticKernel.Test/Configuration/KernelConfigTests.cs @@ -93,22 +93,6 @@ public void SetDefaultHttpRetryConfig() Assert.Equal(httpRetryConfig, config.DefaultHttpRetryConfig); } - [Fact] - public void SetDefaultHttpRetryConfigToNull() - { - // Arrange - var config = new KernelConfig(); - var httpRetryConfig = new KernelConfig.HttpRetryConfig() { MaxRetryCount = 1 }; - config.SetDefaultHttpRetryConfig(httpRetryConfig); - - // Act - config.SetDefaultHttpRetryConfig(null); - - // Assert - Assert.Equal(httpRetryConfig, config.DefaultHttpRetryConfig); - } - - // } [Fact] public void SetDefaultHttpRetryConfigToDefaultIfNotSet() { diff --git a/dotnet/src/SemanticKernel/AI/OpenAI/Clients/AzureOpenAIClientAbstract.cs b/dotnet/src/SemanticKernel/AI/OpenAI/Clients/AzureOpenAIClientAbstract.cs index a7e3450ec2bb..3ac693ed051f 100644 --- a/dotnet/src/SemanticKernel/AI/OpenAI/Clients/AzureOpenAIClientAbstract.cs +++ b/dotnet/src/SemanticKernel/AI/OpenAI/Clients/AzureOpenAIClientAbstract.cs @@ -24,10 +24,9 @@ public abstract class AzureOpenAIClientAbstract : OpenAIClientAbstract /// /// Azure OpenAI API version /// - /// protected string AzureOpenAIApiVersion { - get { return this._azureOpenAIApiVersion; } + get => this._azureOpenAIApiVersion; set { if (string.IsNullOrWhiteSpace(value)) @@ -164,5 +163,5 @@ protected async Task CacheDeploymentsAsync() private string _azureOpenAIApiVersion = DefaultAzureAPIVersion; - #endregion + #endregion private ================================================================================ }