Improve thread pool performance on Unix

src/System.Private.CoreLib/src/System.Private.CoreLib.csproj

@@ -268,11 +268,12 @@
     <Compile Include="System\Threading\ManagedThreadId.cs" />
     <Compile Include="System\Threading\Lock.cs" />
     <Compile Include="System\Threading\Condition.cs" />
-    <Compile Include="System\Threading\FirstLevelSpinWaiter.cs" />
     <Compile Include="System\Threading\Interlocked.cs" />
     <Compile Include="System\Threading\LockHolder.cs" />
+    <Compile Include="System\Threading\LowLevelLifoSemaphore.cs" />
     <Compile Include="System\Threading\LowLevelLock.cs" />
     <Compile Include="System\Threading\LowLevelMonitor.cs" />
+    <Compile Include="System\Threading\LowLevelSpinWaiter.cs" />
     <Compile Include="System\Threading\Monitor.cs" />
     <Compile Include="System\Threading\ObjectHeader.cs" Condition="'$(UseSyncTable)' == 'true'" />
     <Compile Include="System\Threading\Overlapped.cs" />

src/System.Private.CoreLib/src/System/Threading/ClrThreadPool.WorkerThread.cs

@@ -3,7 +3,6 @@
 // See the LICENSE file in the project root for more information.
 
 using System.Globalization;
-using Internal.LowLevelLinq;
 using Internal.Runtime.Augments;
 
 namespace System.Threading
@@ -18,8 +17,16 @@ private static class WorkerThread
             /// <summary>
             /// Semaphore for controlling how many threads are currently working.
             /// </summary>
-            private static LowLevelLifoSemaphore s_semaphore = new LowLevelLifoSemaphore(0, MaxPossibleThreadCount);
-
+            private static LowLevelLifoSemaphore s_semaphore = new LowLevelLifoSemaphore(0, MaxPossibleThreadCount, SemaphoreSpinCount);
+
+            /// <summary>
+            /// Maximum number of spins a thread pool worker thread performs before waiting for work
+            /// </summary>
+            private static int SemaphoreSpinCount
+            {
+                get => AppContextConfigHelper.GetInt16Config("ThreadPool_UnfairSemaphoreSpinLimit", 70, false);
+            }
+
             private static void WorkerThreadStart()
             {
                 ClrThreadPoolEventSource.Log.WorkerThreadStart(ThreadCounts.VolatileReadCounts(ref ThreadPoolInstance._separated.counts).numExistingThreads);

src/System.Private.CoreLib/src/System/Threading/LowLevelLifoSemaphore.Unix.cs

@@ -2,33 +2,35 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
+using System.Collections.Generic;
+
 namespace System.Threading
 {
     /// <summary>
     /// A LIFO semaphore.
     /// Waits on this semaphore are uninterruptible.
     /// </summary>
-    internal sealed class LowLevelLifoSemaphore : IDisposable
+    internal sealed partial class LowLevelLifoSemaphore : IDisposable
     {
         private WaitSubsystem.WaitableObject _semaphore;
 
-        public LowLevelLifoSemaphore(int initialSignalCount, int maximumSignalCount)
+        private void Create(int maximumSignalCount)
         {
-            _semaphore = WaitSubsystem.WaitableObject.NewSemaphore(initialSignalCount, maximumSignalCount);
+            _semaphore = WaitSubsystem.WaitableObject.NewSemaphore(0, maximumSignalCount);
         }
 
-        public bool Wait(int timeoutMs)
+        public void Dispose()
         {
-            return WaitSubsystem.Wait(_semaphore, timeoutMs, false, true);
         }
 
-        public int Release(int count)
+        private bool WaitCore(int timeoutMs)
         {
-            return WaitSubsystem.ReleaseSemaphore(_semaphore, count);
+            return WaitSubsystem.Wait(_semaphore, timeoutMs, false, true);
         }
 
-        public void Dispose()
+        private void ReleaseCore(int count)
         {
+            WaitSubsystem.ReleaseSemaphore(_semaphore, count);
         }
     }
 }

src/System.Private.CoreLib/src/System/Threading/LowLevelLifoSemaphore.Windows.cs

@@ -15,15 +15,13 @@ namespace System.Threading
     /// See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365198(v=vs.85).aspx under How I/O Completion Ports Work.
     /// From the docs "Threads that block their execution on an I/O completion port are released in last-in-first-out (LIFO) order."
     /// </remarks>
-    internal sealed class LowLevelLifoSemaphore : IDisposable
+    internal sealed partial class LowLevelLifoSemaphore : IDisposable
     {
         private IntPtr _completionPort;
 
-        public LowLevelLifoSemaphore(int initialSignalCount, int maximumSignalCount)
+        private void Create(int maximumSignalCount)
         {
-            Debug.Assert(initialSignalCount >= 0, "Windows LowLevelLifoSemaphore does not support a negative signal count"); // TODO: Track actual signal count to enable this
             Debug.Assert(maximumSignalCount > 0);
-            Debug.Assert(initialSignalCount <= maximumSignalCount);
 
             _completionPort =
                 Interop.Kernel32.CreateIoCompletionPort(new IntPtr(-1), IntPtr.Zero, UIntPtr.Zero, maximumSignalCount);
@@ -34,7 +32,6 @@ public LowLevelLifoSemaphore(int initialSignalCount, int maximumSignalCount)
                 exception.HResult = error;
                 throw exception;
             }
-            Release(initialSignalCount);
         }
 
         ~LowLevelLifoSemaphore()
@@ -45,7 +42,7 @@ public LowLevelLifoSemaphore(int initialSignalCount, int maximumSignalCount)
             }
         }
 
-        public bool Wait(int timeoutMs)
+        public bool WaitCore(int timeoutMs)
         {
             Debug.Assert(timeoutMs >= -1);
 
@@ -54,7 +51,7 @@ public bool Wait(int timeoutMs)
             return success;
         }
 
-        public int Release(int count)
+        public void ReleaseCore(int count)
         {
             Debug.Assert(count > 0);
 
@@ -68,7 +65,6 @@ public int Release(int count)
                     throw exception;
                 }
             }
-            return 0; // TODO: Track actual signal count to calculate this
         }
 
         public void Dispose()