From 073f2ba7595e4b28a4326d9a5591e864b1898086 Mon Sep 17 00:00:00 2001 From: AR-May <67507805+AR-May@users.noreply.github.com> Date: Wed, 5 Oct 2022 16:47:33 +0200 Subject: [PATCH 1/4] Add the try-catch block for mutex connection timeout exception. --- src/Build/BackEnd/Client/MSBuildClient.cs | 53 +++++++++++-------- .../BackEnd/Client/MSBuildClientExitType.cs | 9 +++- src/MSBuild/MSBuildClientApp.cs | 1 + 3 files changed, 41 insertions(+), 22 deletions(-) diff --git a/src/Build/BackEnd/Client/MSBuildClient.cs b/src/Build/BackEnd/Client/MSBuildClient.cs index 523cb72e62d..bbd1f3da49b 100644 --- a/src/Build/BackEnd/Client/MSBuildClient.cs +++ b/src/Build/BackEnd/Client/MSBuildClient.cs @@ -8,7 +8,6 @@ using System.Globalization; using System.IO; using System.IO.Pipes; -using System.Linq; using System.Threading; using Microsoft.Build.BackEnd; using Microsoft.Build.BackEnd.Client; @@ -165,33 +164,45 @@ public MSBuildClientExitResult Execute(CancellationToken cancellationToken) #endif CommunicationsUtilities.Trace("Executing build with command line '{0}'", descriptiveCommandLine); - bool serverIsAlreadyRunning = ServerIsRunning(); - if (KnownTelemetry.BuildTelemetry != null) - { - KnownTelemetry.BuildTelemetry.InitialServerState = serverIsAlreadyRunning ? "hot" : "cold"; - } - if (!serverIsAlreadyRunning) + + try { - CommunicationsUtilities.Trace("Server was not running. Starting server now."); - if (!TryLaunchServer()) + bool serverIsAlreadyRunning = ServerIsRunning(); + if (KnownTelemetry.BuildTelemetry != null) + { + KnownTelemetry.BuildTelemetry.InitialServerState = serverIsAlreadyRunning ? "hot" : "cold"; + } + if (!serverIsAlreadyRunning) + { + CommunicationsUtilities.Trace("Server was not running. Starting server now."); + if (!TryLaunchServer()) + { + _exitResult.MSBuildClientExitType = MSBuildClientExitType.LaunchError; + return _exitResult; + } + } + + // Check that server is not busy. + bool serverWasBusy = ServerWasBusy(); + if (serverWasBusy) { - _exitResult.MSBuildClientExitType = MSBuildClientExitType.LaunchError; + CommunicationsUtilities.Trace("Server is busy, falling back to former behavior."); + _exitResult.MSBuildClientExitType = MSBuildClientExitType.ServerBusy; return _exitResult; } - } - // Check that server is not busy. - bool serverWasBusy = ServerWasBusy(); - if (serverWasBusy) - { - CommunicationsUtilities.Trace("Server is busy, falling back to former behavior."); - _exitResult.MSBuildClientExitType = MSBuildClientExitType.ServerBusy; - return _exitResult; + // Connect to server. + if (!TryConnectToServer(serverIsAlreadyRunning ? 1_000 : 20_000)) + { + return _exitResult; + } } - - // Connect to server. - if (!TryConnectToServer(serverIsAlreadyRunning ? 1_000 : 20_000)) + catch (IOException ex) when (ex is not PathTooLongException) { + // For unknown root cause, Mutex.TryOpenExisting can sometimes throw 'Connection timed out' exception preventing to obtain the build server state through it (Running or not, Busy or not). + // See: https://github.com/dotnet/msbuild/issues/7993 + CommunicationsUtilities.Trace("Failed to obtain the current build server state: {0}", ex); + _exitResult.MSBuildClientExitType = MSBuildClientExitType.UnknownServerState; return _exitResult; } diff --git a/src/Build/BackEnd/Client/MSBuildClientExitType.cs b/src/Build/BackEnd/Client/MSBuildClientExitType.cs index e9916bd5414..a2d683cfe40 100644 --- a/src/Build/BackEnd/Client/MSBuildClientExitType.cs +++ b/src/Build/BackEnd/Client/MSBuildClientExitType.cs @@ -24,6 +24,13 @@ public enum MSBuildClientExitType /// The build stopped unexpectedly, for example, /// because a named pipe between the server and the client was unexpectedly closed. /// - Unexpected + Unexpected, + /// + /// The client is not able to identify the server state. + /// + /// + /// This may happen when mutex that is regulating the server state throws. + /// + UnknownServerState } } diff --git a/src/MSBuild/MSBuildClientApp.cs b/src/MSBuild/MSBuildClientApp.cs index 9177f76aa19..0fd4ea40181 100644 --- a/src/MSBuild/MSBuildClientApp.cs +++ b/src/MSBuild/MSBuildClientApp.cs @@ -76,6 +76,7 @@ public static MSBuildApp.ExitType Execute( if (exitResult.MSBuildClientExitType == MSBuildClientExitType.ServerBusy || exitResult.MSBuildClientExitType == MSBuildClientExitType.UnableToConnect || + exitResult.MSBuildClientExitType == MSBuildClientExitType.UnknownServerState || exitResult.MSBuildClientExitType == MSBuildClientExitType.LaunchError) { if (KnownTelemetry.BuildTelemetry != null) From 133463e3c16b6ceacc629a8565be63445e6e1932 Mon Sep 17 00:00:00 2001 From: AR-May <67507805+AR-May@users.noreply.github.com> Date: Wed, 5 Oct 2022 17:28:32 +0200 Subject: [PATCH 2/4] Change the exit type for mutex connection timeout exception in TryLaunchServer. --- src/Build/BackEnd/Client/MSBuildClient.cs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/Build/BackEnd/Client/MSBuildClient.cs b/src/Build/BackEnd/Client/MSBuildClient.cs index bbd1f3da49b..93a79d52bce 100644 --- a/src/Build/BackEnd/Client/MSBuildClient.cs +++ b/src/Build/BackEnd/Client/MSBuildClient.cs @@ -476,10 +476,12 @@ private bool TrySendPacket(Func packetResolver) private bool TryLaunchServer() { string serverLaunchMutexName = $@"Global\msbuild-server-launch-{_handshake.ComputeHash()}"; + try { // For unknown root cause, opening mutex can sometimes throw 'Connection timed out' exception. See: https://github.com/dotnet/msbuild/issues/7993 using var serverLaunchMutex = ServerNamedMutex.OpenOrCreateMutex(serverLaunchMutexName, out bool mutexCreatedNew); + if (!mutexCreatedNew) { // Some other client process launching a server and setting a build request for it. Fallback to usual msbuild app build. @@ -487,12 +489,20 @@ private bool TryLaunchServer() _exitResult.MSBuildClientExitType = MSBuildClientExitType.ServerBusy; return false; } + } + catch (IOException ex) when (ex is not PathTooLongException) + { + CommunicationsUtilities.Trace("Failed to obtain the current build server state: {0}", ex); + _exitResult.MSBuildClientExitType = MSBuildClientExitType.UnknownServerState; + return false; + } + try + { string[] msBuildServerOptions = new string[] { "/nologo", "/nodemode:8" }; - NodeLauncher nodeLauncher = new NodeLauncher(); CommunicationsUtilities.Trace("Starting Server..."); Process msbuildProcess = nodeLauncher.Start(_msbuildLocation, string.Join(" ", msBuildServerOptions)); From 538fb7503667c3d93744137a4485f842756effef Mon Sep 17 00:00:00 2001 From: AR-May <67507805+AR-May@users.noreply.github.com> Date: Thu, 6 Oct 2022 11:02:19 +0200 Subject: [PATCH 3/4] Address PR comments. --- src/Build/BackEnd/Client/MSBuildClient.cs | 2 +- src/Build/BackEnd/Client/MSBuildClientExitType.cs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Build/BackEnd/Client/MSBuildClient.cs b/src/Build/BackEnd/Client/MSBuildClient.cs index 93a79d52bce..4d894d1bfae 100644 --- a/src/Build/BackEnd/Client/MSBuildClient.cs +++ b/src/Build/BackEnd/Client/MSBuildClient.cs @@ -177,7 +177,7 @@ public MSBuildClientExitResult Execute(CancellationToken cancellationToken) CommunicationsUtilities.Trace("Server was not running. Starting server now."); if (!TryLaunchServer()) { - _exitResult.MSBuildClientExitType = MSBuildClientExitType.LaunchError; + _exitResult.MSBuildClientExitType = (_exitResult.MSBuildClientExitType == MSBuildClientExitType.Success) ? MSBuildClientExitType.LaunchError : _exitResult.MSBuildClientExitType; return _exitResult; } } diff --git a/src/Build/BackEnd/Client/MSBuildClientExitType.cs b/src/Build/BackEnd/Client/MSBuildClientExitType.cs index a2d683cfe40..9ac0d49652a 100644 --- a/src/Build/BackEnd/Client/MSBuildClientExitType.cs +++ b/src/Build/BackEnd/Client/MSBuildClientExitType.cs @@ -30,6 +30,7 @@ public enum MSBuildClientExitType /// /// /// This may happen when mutex that is regulating the server state throws. + /// See: https://github.com/dotnet/msbuild/issues/7993. /// UnknownServerState } From 228639686a3d6c6e1a71abac60a8286a38ac63a4 Mon Sep 17 00:00:00 2001 From: AR-May <67507805+AR-May@users.noreply.github.com> Date: Fri, 7 Oct 2022 11:53:33 +0200 Subject: [PATCH 4/4] Address PR comments - 2. --- src/Build/BackEnd/Client/MSBuildClient.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Build/BackEnd/Client/MSBuildClient.cs b/src/Build/BackEnd/Client/MSBuildClient.cs index 4d894d1bfae..f7b4a22cfbf 100644 --- a/src/Build/BackEnd/Client/MSBuildClient.cs +++ b/src/Build/BackEnd/Client/MSBuildClient.cs @@ -202,6 +202,7 @@ public MSBuildClientExitResult Execute(CancellationToken cancellationToken) // For unknown root cause, Mutex.TryOpenExisting can sometimes throw 'Connection timed out' exception preventing to obtain the build server state through it (Running or not, Busy or not). // See: https://github.com/dotnet/msbuild/issues/7993 CommunicationsUtilities.Trace("Failed to obtain the current build server state: {0}", ex); + CommunicationsUtilities.Trace("HResult: {0}.", ex.HResult); _exitResult.MSBuildClientExitType = MSBuildClientExitType.UnknownServerState; return _exitResult; } @@ -493,6 +494,7 @@ private bool TryLaunchServer() catch (IOException ex) when (ex is not PathTooLongException) { CommunicationsUtilities.Trace("Failed to obtain the current build server state: {0}", ex); + CommunicationsUtilities.Trace("HResult: {0}.", ex.HResult); _exitResult.MSBuildClientExitType = MSBuildClientExitType.UnknownServerState; return false; }