From 69978b7f49454344e6b954c039dbc06cab3f7a71 Mon Sep 17 00:00:00 2001 From: antonfirsov Date: Fri, 2 Feb 2024 16:49:09 +0100 Subject: [PATCH 1/7] send a connection WINDOW_UPDATE before RTT PING --- .../SocketsHttpHandler/Http2Connection.cs | 34 +++++++++++-------- .../Http2StreamWindowManager.cs | 27 +++++++++++---- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs index 45dc67d7342c52..d688d36cf7fde6 100644 --- a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs +++ b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs @@ -640,7 +640,7 @@ private async ValueTask ProcessHeadersFrame(FrameHeader frameHeader) if (http2Stream != null) { http2Stream.OnHeadersStart(); - _rttEstimator.OnDataOrHeadersReceived(this); + _rttEstimator.OnDataOrHeadersReceived(this, sendWindowUpdateBeforePing: true); headersHandler = http2Stream; } else @@ -769,21 +769,16 @@ private void ProcessDataFrame(FrameHeader frameHeader) ReadOnlySpan frameData = GetFrameData(_incomingBuffer.ActiveSpan.Slice(0, frameHeader.PayloadLength), hasPad: frameHeader.PaddedFlag, hasPriority: false); - if (http2Stream != null) - { - bool endStream = frameHeader.EndStreamFlag; - - http2Stream.OnResponseData(frameData, endStream); - - if (!endStream && frameData.Length > 0) - { - _rttEstimator.OnDataOrHeadersReceived(this); - } - } + bool endStream = frameHeader.EndStreamFlag; + http2Stream?.OnResponseData(frameData, endStream); if (frameData.Length > 0) { - ExtendWindow(frameData.Length); + bool windowUpdateSent = ExtendWindow(frameData.Length); + if (!endStream) + { + _rttEstimator.OnDataOrHeadersReceived(this, sendWindowUpdateBeforePing: !windowUpdateSent); + } } _incomingBuffer.Discard(frameHeader.PayloadLength); @@ -1770,7 +1765,7 @@ private Task SendWindowUpdateAsync(int streamId, int amount) }); } - private void ExtendWindow(int amount) + private bool ExtendWindow(int amount) { if (NetEventSource.Log.IsEnabled()) Trace($"{nameof(amount)}={amount}"); Debug.Assert(amount > 0); @@ -1784,7 +1779,7 @@ private void ExtendWindow(int amount) if (_pendingWindowUpdate < ConnectionWindowThreshold) { if (NetEventSource.Log.IsEnabled()) Trace($"{nameof(_pendingWindowUpdate)} {_pendingWindowUpdate} < {ConnectionWindowThreshold}."); - return; + return false; } windowUpdateSize = _pendingWindowUpdate; @@ -1792,6 +1787,15 @@ private void ExtendWindow(int amount) } LogExceptions(SendWindowUpdateAsync(0, windowUpdateSize)); + return true; + } + + private bool ForceSendConnectionWindowUpdate() + { + if (_pendingWindowUpdate == 0) return false; + LogExceptions(SendWindowUpdateAsync(0, _pendingWindowUpdate)); + _pendingWindowUpdate = 0; + return true; } public override long GetIdleTicks(long nowTicks) diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2StreamWindowManager.cs b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2StreamWindowManager.cs index e5dd33aa9fd361..470cacbe2de34c 100644 --- a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2StreamWindowManager.cs +++ b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2StreamWindowManager.cs @@ -138,13 +138,18 @@ private void AdjustWindowDynamic(int bytesConsumed, Http2Stream stream) // Assuming that the network characteristics of the connection wouldn't change much within its lifetime, we are maintaining a running minimum value. // The more PINGs we send, the more accurate is the estimation of MinRtt, however we should be careful not to send too many of them, // to avoid triggering the server's PING flood protection which may result in an unexpected GOAWAY. - // With most servers we are fine to send PINGs, as long as we are reading their data, this rule is well formalized for gRPC: + // + // Several strategies have been implemented to conform with real life servers. + // 1. With most servers we are fine to send PINGs as long as we are reading their data, a rule formalized by a gRPC spec: // https://github.com/grpc/proposal/blob/master/A8-client-side-keepalive.md - // As a rule of thumb, we can send send a PING whenever we receive DATA or HEADERS, however, there are some servers which allow receiving only - // a limited amount of PINGs within a given timeframe. - // To deal with the conflicting requirements: - // - We send an initial burst of 'InitialBurstCount' PINGs, to get a relatively good estimation fast - // - Afterwards, we send PINGs with the maximum frequency of 'PingIntervalInSeconds' PINGs per second + // According to this rule, we are OK to send a PING whenever we receive DATA or HEADERS, since the servers conforming to this doc + // will reset their unsolicited ping counter whenever they *send* DATA or HEADERS. + // 2. Some servers allow receiving only a limited amount of PINGs within a given timeframe. + // To deal with this, we send an initial burst of 'InitialBurstCount' (=4) PINGs, to get a relatively good estimation fast. Afterwards, + // we send PINGs each 'PingIntervalInSeconds' second, to maintain our estimation without triggering these servers. + // 3. Some servers in Google's backends reset their unsolicited ping counter when they *receive* DATA, HEADERS, or WINDOW_UPDATE. + // To deal with this, we need to make sure to send a connection WINDOW_UPDATE before sending a PING. The initial burst is an exception + // to this rule, since the mentioned server can tolerate 4 PINGs without receiving a WINDOW_UPDATE. // // Threading: // OnInitialSettingsSent() is called during initialization, all other methods are triggered by HttpConnection.ProcessIncomingFramesAsync(), @@ -194,7 +199,7 @@ internal void OnInitialSettingsAckReceived(Http2Connection connection) _state = State.Waiting; } - internal void OnDataOrHeadersReceived(Http2Connection connection) + internal void OnDataOrHeadersReceived(Http2Connection connection, bool sendWindowUpdateBeforePing) { if (_state != State.Waiting) return; @@ -204,6 +209,14 @@ internal void OnDataOrHeadersReceived(Http2Connection connection) { if (initial) _initialBurst--; + // When sendWindowUpdateBeforePing is true, try to send a WINDOW_UPDATE to make Google backends happy. + // Unless we are doing the initial burst, do not send PING if we were not able to send the WINDOW_UPDATE. + // See point 3. in the comments above the class definition for more info. + if (sendWindowUpdateBeforePing && !connection.ForceSendConnectionWindowUpdate() && !initial) + { + return; + } + // Send a PING _pingCounter--; if (NetEventSource.Log.IsEnabled()) connection.Trace($"[FlowControl] Sending RTT PING with payload {_pingCounter}"); From 61fcf4757a0f7357ed1f249d7a98eb67d17a2795 Mon Sep 17 00:00:00 2001 From: antonfirsov Date: Fri, 2 Feb 2024 17:04:47 +0100 Subject: [PATCH 2/7] adjust tests --- .../tests/FunctionalTests/HttpClientHandlerTest.Http2.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libraries/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.Http2.cs b/src/libraries/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.Http2.cs index 7b14f99393c282..7f2f3651e98252 100644 --- a/src/libraries/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.Http2.cs +++ b/src/libraries/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.Http2.cs @@ -2431,6 +2431,7 @@ public async Task PostAsyncDuplex_ClientSendsEndStream_Success() HttpResponseMessage response = await responseTask; Stream responseStream = await response.Content.ReadAsStreamAsync(); + connection.IgnoreWindowUpdates(); // Send some data back and forth await SendAndReceiveResponseDataAsync(contentBytes, responseStream, connection, streamId); await SendAndReceiveResponseDataAsync(contentBytes, responseStream, connection, streamId); @@ -2491,6 +2492,7 @@ public async Task PostAsyncDuplex_ServerSendsEndStream_Success() HttpResponseMessage response = await responseTask; Stream responseStream = await response.Content.ReadAsStreamAsync(); + connection.IgnoreWindowUpdates(); // Send some data back and forth await SendAndReceiveResponseDataAsync(contentBytes, responseStream, connection, streamId); await SendAndReceiveResponseDataAsync(contentBytes, responseStream, connection, streamId); From f9d334c63e5f2f2cc582d32f3e9321c5c36aa0ee Mon Sep 17 00:00:00 2001 From: antonfirsov Date: Tue, 6 Feb 2024 18:21:51 +0100 Subject: [PATCH 3/7] add test --- ...SocketsHttpHandlerTest.Http2FlowControl.cs | 43 +++++++++++++++---- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.Http2FlowControl.cs b/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.Http2FlowControl.cs index 4862c0a4ae52c5..f1dc09a9f7358f 100644 --- a/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.Http2FlowControl.cs +++ b/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.Http2FlowControl.cs @@ -128,7 +128,7 @@ static async Task RunTest() TimeSpan.FromMilliseconds(30), TimeSpan.Zero, 2 * 1024 * 1024, - null); + maxWindowForPingStopValidation: MaxWindow); Assert.True(maxCredit <= MaxWindow); } @@ -181,19 +181,34 @@ static async Task RunTest() RemoteExecutor.Invoke(RunTest, options).Dispose(); } + [OuterLoop("Runs long")] + [Fact] + public async Task LongRunningSlowServerStream_NoInvalidPingsAreSent() + { + // A scenario similar to https://github.com/grpc/grpc-dotnet/issues/2361. + // We need to send a small amount of data so the connection window is not consumed and no "standard" WINDOW_UPDATEs are sent and + // we also need to do it very slowly to cover some RTT PINGs after the initial burst. + // This scenario should trigger the "forced WINDOW_UPDATE" logic in the implementation, ensuring that no more than 4 PINGs are sent without a WINDOW_UPDATE. + await TestClientWindowScalingAsync( + TimeSpan.FromMilliseconds(500), + TimeSpan.FromMilliseconds(500), + 1024, + _output, + dataPerFrame: 32); + } + private static async Task TestClientWindowScalingAsync( TimeSpan networkDelay, TimeSpan slowBandwidthSimDelay, int bytesToDownload, ITestOutputHelper output = null, - int maxWindowForPingStopValidation = int.MaxValue, // set to actual maximum to test if we stop sending PING when window reached maximum - Action configureHandler = null) + int dataPerFrame = 16384, + int maxWindowForPingStopValidation = 16 * 1024 * 1024) // set to actual maximum to test if we stop sending PING when window reached maximum { TimeSpan timeout = TimeSpan.FromSeconds(30); CancellationTokenSource timeoutCts = new CancellationTokenSource(timeout); HttpClientHandler handler = CreateHttpClientHandler(HttpVersion20.Value); - configureHandler?.Invoke(GetUnderlyingSocketsHttpHandler(handler)); using Http2LoopbackServer server = Http2LoopbackServer.CreateServer(NoAutoPingResponseHttp2Options); using HttpClient client = new HttpClient(handler, true); @@ -225,13 +240,13 @@ private static async Task TestClientWindowScalingAsync( using SemaphoreSlim writeSemaphore = new SemaphoreSlim(1); int remainingBytes = bytesToDownload; - bool pingReceivedAfterReachingMaxWindow = false; + string unexpectedPingReason = null; bool unexpectedFrameReceived = false; CancellationTokenSource stopFrameProcessingCts = new CancellationTokenSource(); CancellationTokenSource linkedCts = CancellationTokenSource.CreateLinkedTokenSource(stopFrameProcessingCts.Token, timeoutCts.Token); Task processFramesTask = ProcessIncomingFramesAsync(linkedCts.Token); - byte[] buffer = new byte[16384]; + byte[] buffer = new byte[dataPerFrame]; while (remainingBytes > 0) { @@ -259,7 +274,7 @@ private static async Task TestClientWindowScalingAsync( int dataReceived = (await response.Content.ReadAsByteArrayAsync()).Length; Assert.Equal(bytesToDownload, dataReceived); - Assert.False(pingReceivedAfterReachingMaxWindow, "Server received a PING after reaching max window"); + Assert.Null(unexpectedPingReason); Assert.False(unexpectedFrameReceived, "Server received an unexpected frame, see test output for more details."); return maxCredit; @@ -270,6 +285,7 @@ async Task ProcessIncomingFramesAsync(CancellationToken cancellationToken) // We should not receive any more RTT PING's after this point int maxWindowCreditThreshold = (int) (0.9 * maxWindowForPingStopValidation); output?.WriteLine($"maxWindowCreditThreshold: {maxWindowCreditThreshold} maxWindowForPingStopValidation: {maxWindowForPingStopValidation}"); + int pingsWithoutWindowUpdate = 0; try { @@ -284,10 +300,18 @@ async Task ProcessIncomingFramesAsync(CancellationToken cancellationToken) output?.WriteLine($"Received PING ({pingFrame.Data})"); + pingsWithoutWindowUpdate++; if (maxCredit > maxWindowCreditThreshold) { - output?.WriteLine("PING was unexpected"); - Volatile.Write(ref pingReceivedAfterReachingMaxWindow, true); + Volatile.Write(ref unexpectedPingReason, "The server received a PING after reaching max window"); + output?.WriteLine($"PING was unexpected: {unexpectedPingReason}"); + } + + // Exceeding this limit may trigger a GOAWAY on some servers. See implementation comments for more details. + if (pingsWithoutWindowUpdate > 4) + { + Volatile.Write(ref unexpectedPingReason, $"The server received {pingsWithoutWindowUpdate} PINGs without receiving a WINDOW_UPDATE"); + output?.WriteLine($"PING was unexpected: {unexpectedPingReason}"); } await writeSemaphore.WaitAsync(cancellationToken); @@ -296,6 +320,7 @@ async Task ProcessIncomingFramesAsync(CancellationToken cancellationToken) } else if (frame is WindowUpdateFrame windowUpdateFrame) { + pingsWithoutWindowUpdate = 0; // Ignore connection window: if (windowUpdateFrame.StreamId != streamId) continue; From aa896fc8ad31382938b4c9b41bcf8aa091ccffe1 Mon Sep 17 00:00:00 2001 From: antonfirsov Date: Tue, 6 Feb 2024 18:43:56 +0100 Subject: [PATCH 4/7] add logging to ForceSendConnectionWindowUpdate --- .../src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs index d688d36cf7fde6..d5d8eb7d63c868 100644 --- a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs +++ b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs @@ -1792,7 +1792,9 @@ private bool ExtendWindow(int amount) private bool ForceSendConnectionWindowUpdate() { + if (NetEventSource.Log.IsEnabled()) Trace($"{nameof(_pendingWindowUpdate)}={_pendingWindowUpdate}"); if (_pendingWindowUpdate == 0) return false; + LogExceptions(SendWindowUpdateAsync(0, _pendingWindowUpdate)); _pendingWindowUpdate = 0; return true; From 574def5fde2c7689acc07491a4efd30514899e03 Mon Sep 17 00:00:00 2001 From: antonfirsov Date: Tue, 13 Feb 2024 16:27:03 +0100 Subject: [PATCH 5/7] http2Stream must not be null for OnDataOrHeadersReceived() --- .../src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs index d5d8eb7d63c868..79fefc5ce4a54d 100644 --- a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs +++ b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs @@ -775,7 +775,7 @@ private void ProcessDataFrame(FrameHeader frameHeader) if (frameData.Length > 0) { bool windowUpdateSent = ExtendWindow(frameData.Length); - if (!endStream) + if (http2Stream is not null && !endStream) { _rttEstimator.OnDataOrHeadersReceived(this, sendWindowUpdateBeforePing: !windowUpdateSent); } From 69c11de722d8363173779a8b68a65a3f47eb73b9 Mon Sep 17 00:00:00 2001 From: antonfirsov Date: Tue, 13 Feb 2024 16:52:36 +0100 Subject: [PATCH 6/7] fix PostAsyncDuplex_DisposeResponseBodyAfterEndReceivedButBeforeConsumed_ResetsStreamAndThrowsOnRequestStreamWriteAndResponseStreamRead... --- .../tests/FunctionalTests/HttpClientHandlerTest.Http2.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libraries/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.Http2.cs b/src/libraries/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.Http2.cs index 7f2f3651e98252..cecae782483099 100644 --- a/src/libraries/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.Http2.cs +++ b/src/libraries/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.Http2.cs @@ -2836,6 +2836,7 @@ public async Task PostAsyncDuplex_DisposeResponseBodyBeforeEnd_ResetsStreamAndTh Task responseTask = client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); connection = await server.EstablishConnectionAsync(); + connection.IgnoreWindowUpdates(); // Client should have sent the request headers, and the request stream should now be available Stream requestStream = await duplexContent.WaitForStreamAsync(); From 24c1b1a5aa80f330cb75c16bbcb6e39872bcb2f2 Mon Sep 17 00:00:00 2001 From: antonfirsov Date: Tue, 13 Feb 2024 17:38:17 +0100 Subject: [PATCH 7/7] fix more outerloop tests --- .../tests/FunctionalTests/HttpClientHandlerTest.Http2.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.Http2.cs b/src/libraries/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.Http2.cs index cecae782483099..02644d38896019 100644 --- a/src/libraries/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.Http2.cs +++ b/src/libraries/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.Http2.cs @@ -2836,7 +2836,6 @@ public async Task PostAsyncDuplex_DisposeResponseBodyBeforeEnd_ResetsStreamAndTh Task responseTask = client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); connection = await server.EstablishConnectionAsync(); - connection.IgnoreWindowUpdates(); // Client should have sent the request headers, and the request stream should now be available Stream requestStream = await duplexContent.WaitForStreamAsync(); @@ -2876,6 +2875,7 @@ public async Task PostAsyncDuplex_DisposeResponseBodyBeforeEnd_ResetsStreamAndTh // This allows the request processing to complete. duplexContent.Fail(e); + connection.IgnoreWindowUpdates(); // The RTT algorithm may send a WINDOW_UPDATE before RST_STREAM. // Client should set RST_STREAM. await connection.ReadRstStreamAsync(streamId); } @@ -2949,6 +2949,7 @@ public async Task PostAsyncDuplex_DisposeResponseBodyAfterEndReceivedButBeforeCo // This allows the request processing to complete. duplexContent.Fail(e); + connection.IgnoreWindowUpdates(); // The RTT algorithm may send a WINDOW_UPDATE before RST_STREAM. // Client should set RST_STREAM. await connection.ReadRstStreamAsync(streamId); }