Migrate coreclr's worker thread pool to be able to use the portable thread pool in opt-in fashion

src/coreclr/src/System.Private.CoreLib/System.Private.CoreLib.csproj

@@ -20,6 +20,9 @@
     <ILLinkTrimAssembly>true</ILLinkTrimAssembly>
     <ILLinkTrimXml>$(IntermediateOutputPath)System.Private.CoreLib.xml</ILLinkTrimXml>
     <ILLinkDirectory>$(MSBuildThisFileDirectory)src\ILLink\</ILLinkDirectory>
+
+    <FeaturePortableThreadPool>true</FeaturePortableThreadPool>
+    <FeatureSharedLowLevelLock>true</FeatureSharedLowLevelLock>
   </PropertyGroup>
 
   <!-- Note that various places in SPCL depend on this resource name i.e. TplEventSource -->
@@ -289,6 +292,7 @@
     <Compile Include="$(BclSourcesRoot)\Interop\Unix\Interop.Libraries.cs" />
     <Compile Include="$(BclSourcesRoot)\System\DateTime.Unix.CoreCLR.cs" />
     <Compile Include="$(BclSourcesRoot)\System\Threading\ClrThreadPoolBoundHandle.Unix.cs" />
+    <Compile Include="$(BclSourcesRoot)\System\Threading\LowLevelLifoSemaphore.Unix.cs" />
   </ItemGroup>
   <ItemGroup Condition="'$(TargetsWindows)' == 'true'">
     <Compile Include="$(BclSourcesRoot)\System\DateTime.Windows.CoreCLR.cs" />

src/coreclr/src/System.Private.CoreLib/src/System/Threading/LowLevelLifoSemaphore.Unix.cs

@@ -0,0 +1,51 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using Microsoft.Win32.SafeHandles;
+
+namespace System.Threading
+{
+    /// <summary>
+    /// A LIFO semaphore implemented using the PAL's semaphore with uninterruptible waits.
+    /// </summary>
+    internal sealed partial class LowLevelLifoSemaphore : IDisposable
+    {
+        private Semaphore? _semaphore;
+
+        private void Create(int maximumSignalCount)
+        {
+            Debug.Assert(maximumSignalCount > 0);
+            _semaphore = new Semaphore(0, maximumSignalCount);
+        }
+
+        public bool WaitCore(int timeoutMs)
+        {
+            Debug.Assert(_semaphore != null);
+            Debug.Assert(timeoutMs >= -1);
+
+            int waitResult = WaitNative(_semaphore!.SafeWaitHandle, timeoutMs);
+            Debug.Assert(waitResult == WaitHandle.WaitSuccess || waitResult == WaitHandle.WaitTimeout);
+            return waitResult == WaitHandle.WaitSuccess;
+        }
+
+        [DllImport(RuntimeHelpers.QCall, CharSet = CharSet.Unicode)]
+        private static extern int WaitNative(SafeWaitHandle handle, int timeoutMs);
+
+        public void ReleaseCore(int count)
+        {
+            Debug.Assert(_semaphore != null);
+            Debug.Assert(count > 0);
+
+            _semaphore!.Release(count);
+        }
+
+        public void Dispose()
+        {
+            Debug.Assert(_semaphore != null);
+            _semaphore!.Dispose();
+        }
+    }
+}

src/coreclr/src/System.Private.CoreLib/src/System/Threading/Thread.CoreCLR.cs

@@ -144,6 +144,11 @@ public sealed partial class Thread
         private int _managedThreadId; // INT32
 #pragma warning restore CA1823, 169
 
+        // This is used for a quick check on thread pool threads after running a work item to determine if the name, background
+        // state, or priority were changed by the work item, and if so to reset it. Other threads may also change some of those,
+        // but those types of changes may race with the reset anyway, so this field doesn't need to be synchronized.
+        private bool _mayNeedResetForThreadPool;
+
         private Thread() { }
 
         private void Create(ThreadStart start) =>
@@ -259,6 +264,9 @@ private void SetCultureOnUnstartedThreadNoCheck(CultureInfo value, bool uiCultur
 
         public static void Sleep(int millisecondsTimeout) => SleepInternal(millisecondsTimeout);
 
+        [DllImport(RuntimeHelpers.QCall)]
+        internal static extern void UninterruptibleSleep0();
+
         /// <summary>
         /// Wait for a length of time proportional to 'iterations'.  Each iteration is should
         /// only take a few machine instructions.  Calling this API is preferable to coding
@@ -337,7 +345,14 @@ public extern bool IsAlive
         public bool IsBackground
         {
             get => IsBackgroundNative();
-            set => SetBackgroundNative(value);
+            set
+            {
+                SetBackgroundNative(value);
+                if (!value)
+                {
+                    _mayNeedResetForThreadPool = true;
+                }
+            }
         }
 
         [MethodImpl(MethodImplOptions.InternalCall)]
@@ -351,13 +366,22 @@ public extern bool IsThreadPoolThread
         {
             [MethodImpl(MethodImplOptions.InternalCall)]
             get;
+            [MethodImpl(MethodImplOptions.InternalCall)]
+            internal set;
         }
 
         /// <summary>Returns the priority of the thread.</summary>
         public ThreadPriority Priority
         {
             get => (ThreadPriority)GetPriorityNative();
-            set => SetPriorityNative((int)value);
+            set
+            {
+                SetPriorityNative((int)value);
+                if (value != ThreadPriority.Normal)
+                {
+                    _mayNeedResetForThreadPool = true;
+                }
+            }
         }
 
         [MethodImpl(MethodImplOptions.InternalCall)]
@@ -503,12 +527,23 @@ public static int GetCurrentProcessorId()
         // we will record that in a readonly static so that it could become a JIT constant and bypass caching entirely.
         private static readonly bool s_isProcessorNumberReallyFast = ProcessorIdCache.ProcessorNumberSpeedCheck();
 
-#pragma warning disable CA1822 // Mark members as static
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal void ResetThreadPoolThread()
         {
-            // Currently implemented in unmanaged method Thread::InternalReset and
-            // called internally from the ThreadPool in NotifyWorkItemComplete.
+            Debug.Assert(this == CurrentThread);
+            Debug.Assert(IsThreadPoolThread);
+
+            if (!ThreadPool.UsePortableThreadPool)
+            {
+                // Currently implemented in unmanaged method Thread::InternalReset and
+                // called internally from the ThreadPool in NotifyWorkItemComplete.
+                return;
+            }
+
+            if (_mayNeedResetForThreadPool)
+            {
+                ResetThreadPoolThreadSlow();
+            }
         }
-#pragma warning restore CA1822
     } // End of class Thread
 }