From ce89a9a44830039c7883d2d7bf4f32c110e71426 Mon Sep 17 00:00:00 2001
From: Santiago Fernandez Madero <safern@microsoft.com>
Date: Mon, 18 Nov 2019 14:48:35 -0800
Subject: [PATCH 1/2] Disable tests on CoreCLR using new attribute

---
 NuGet.config                                  |  1 +
 eng/Versions.props                            |  2 +-
 eng/helixcorefxtests.proj                     | 28 +------
 src/coreclr/tests/CoreFX/CoreFX.issues.rsp    | 73 -------------------
 .../tests/scripts/run-corefx-tests.bat        |  1 -
 ...ationBuilderAttributedOverrideUnitTests.cs |  1 +
 .../tests/FunctionalTests/AssemblyInfo.cs     |  7 ++
 .../System.Data.SqlClient.Tests.csproj        |  1 +
 .../tests/AssemblyInfo.cs                     |  7 ++
 .../System.Linq.Expressions.Tests.csproj      |  1 +
 .../tests/FunctionalTests/AssemblyInfo.cs     |  7 ++
 .../System.Net.Http.Functional.Tests.csproj   |  1 +
 .../tests/HttpListenerAuthenticationTests.cs  |  1 +
 .../tests/HttpListenerContextTests.cs         |  1 +
 .../HttpListenerResponseTests.Cookies.cs      |  1 +
 .../tests/HttpListenerResponseTests.cs        |  2 +
 .../tests/Functional/SmtpClientTest.cs        |  1 +
 .../System.Net.Requests/tests/LoggingTest.cs  |  1 +
 .../tests/FunctionalTests/SslStreamSniTest.cs |  1 +
 .../tests/FunctionalTests/AssemblyInfo.cs     |  7 ++
 .../System.Net.Sockets.Tests.csproj           |  1 +
 .../BigInteger/BigIntegerToStringTests.cs     |  1 +
 .../tests/BinaryFormatterTests.cs             |  1 +
 .../tests/Regex.Match.Tests.cs                |  1 +
 ...ystem.Utf8String.Experimental.Tests.csproj |  1 +
 25 files changed, 50 insertions(+), 100 deletions(-)
 delete mode 100644 src/coreclr/tests/CoreFX/CoreFX.issues.rsp
 create mode 100644 src/libraries/System.Data.SqlClient/tests/FunctionalTests/AssemblyInfo.cs
 create mode 100644 src/libraries/System.Linq.Expressions/tests/AssemblyInfo.cs
 create mode 100644 src/libraries/System.Net.Http/tests/FunctionalTests/AssemblyInfo.cs
 create mode 100644 src/libraries/System.Net.Sockets/tests/FunctionalTests/AssemblyInfo.cs

diff --git a/NuGet.config b/NuGet.config
index 6b6e54b554f77f..d8f0d5ea500a69 100644
--- a/NuGet.config
+++ b/NuGet.config
@@ -16,6 +16,7 @@
     <add key="dotnet5-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json" />
     <add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
     <add key="dotnet-coreclr" value="https://dotnetfeed.blob.core.windows.net/dotnet-coreclr/index.json" />
+    <add key="safern-test" value="https://www.myget.org/F/safern-test/api/v3/index.json" />
   </packageSources>
   <disabledPackageSources>
     <clear />
diff --git a/eng/Versions.props b/eng/Versions.props
index b14b34b99ae734..0ec5250c0d7e4a 100644
--- a/eng/Versions.props
+++ b/eng/Versions.props
@@ -37,7 +37,7 @@
     <MicrosoftDotNetCodeAnalysisVersion>5.0.0-beta.19567.2</MicrosoftDotNetCodeAnalysisVersion>
     <MicrosoftDotNetGenAPIVersion>5.0.0-beta.19567.2</MicrosoftDotNetGenAPIVersion>
     <MicrosoftDotNetGenFacadesVersion>5.0.0-beta.19567.2</MicrosoftDotNetGenFacadesVersion>
-    <MicrosoftDotNetXUnitExtensionsVersion>5.0.0-beta.19567.2</MicrosoftDotNetXUnitExtensionsVersion>
+    <MicrosoftDotNetXUnitExtensionsVersion>5.0.0-dev</MicrosoftDotNetXUnitExtensionsVersion>
     <MicrosoftDotNetXUnitConsoleRunnerVersion>2.5.1-beta.19567.2</MicrosoftDotNetXUnitConsoleRunnerVersion>
     <MicrosoftDotNetBuildTasksPackagingVersion>5.0.0-beta.19567.2</MicrosoftDotNetBuildTasksPackagingVersion>
     <MicrosoftDotNetRemoteExecutorVersion>5.0.0-beta.19567.2</MicrosoftDotNetRemoteExecutorVersion>
diff --git a/eng/helixcorefxtests.proj b/eng/helixcorefxtests.proj
index 7b14fb1d0ca70a..5eb26c3a77c31d 100644
--- a/eng/helixcorefxtests.proj
+++ b/eng/helixcorefxtests.proj
@@ -89,15 +89,6 @@
     <TestWorkingDir Condition="'$(__TestWorkingDir)'==''">$(ProjectDir)artifacts\tests\coreclr\$(__BuildOS).$(__BuildArch).$(__BuildType)\</TestWorkingDir>
     <TargetsWindows Condition="'$(__BuildOS)' == 'Windows_NT'">true</TargetsWindows>
 
-    <!-- We have a single, universal exclusion file for all CoreFX test exclusions, for CoreFX tests run
-         in the CoreCLR repo. Note in particular that we don't have a unique exclusion file for each test
-         assembly, or for each processor architecture, or for each operating system, or for each stress
-         scenario type. This means that we generally exclude more tests than is strictly necessary for
-         any particular job. However, this mechanism is super simple; we don't need to manage fine-
-         grained exclusions; and there really shouldn't be any (or many) exclusions anyway.
-    -->
-    <CoreFXTestExclusionFile>$(ProjectDir)src\coreclr\tests\CoreFX\CoreFX.issues.rsp</CoreFXTestExclusionFile>
-
     <TestHostRootPath>$(TestWorkingDir)testhost\</TestHostRootPath>
     <TestArchiveRuntimeRoot>$(TestWorkingDir)helix\</TestArchiveRuntimeRoot>
 
@@ -118,23 +109,10 @@
       The arguments passed in to the run script in are the runtime directory (-r) and the exclusion response file
       passed to xunit (rsp-file).
     -->
-    <HelixCommand Condition="'$(TargetsWindows)' == 'true'">call RunTests.cmd -r %HELIX_CORRELATION_PAYLOAD% --rsp-file %HELIX_CORRELATION_PAYLOAD%\CoreFX.issues.rsp </HelixCommand>
-    <HelixCommand Condition="'$(TargetsWindows)' != 'true'">./RunTests.sh -r $HELIX_CORRELATION_PAYLOAD --rsp-file $HELIX_CORRELATION_PAYLOAD/CoreFX.issues.rsp </HelixCommand>
+    <HelixCommand Condition="'$(TargetsWindows)' == 'true'">call RunTests.cmd -r %HELIX_CORRELATION_PAYLOAD%</HelixCommand>
+    <HelixCommand Condition="'$(TargetsWindows)' != 'true'">./RunTests.sh -r $HELIX_CORRELATION_PAYLOAD</HelixCommand>
   </PropertyGroup>
 
-  <!--
-    Copy the CoreFX test exclusion response file (which is passed to the xunit console runner) to someplace we can
-    access it during the test run. We currently just copy it to the root of the generated .NET Core testhost
-    directory, which we use as the root of the correlation directory.
-  -->
-  <Target Name="CopyRSPFile">
-    <Copy
-      SourceFiles="$(CoreFXTestExclusionFile)"
-      DestinationFolder="$(TestHostRootPath)" />
-
-    <Message Importance="High" Text="Copied $(CoreFXTestExclusionFile) into $(TestHostRootPath)" />
-  </Target>
-
   <PropertyGroup>
     <!-- Set the name of the scenario file. Note that this is only used in invocations where $(Scenario) is set. -->
     <TestEnvFileName Condition=" '$(TargetsWindows)' == 'true' ">SetStressModes_$(Scenario).cmd</TestEnvFileName>
@@ -178,7 +156,7 @@
     This just causes its dependent targets to be run.
   -->
   <Target Name="PrepareCorrelationPayloadDirectory"
-          DependsOnTargets="CopyRSPFile;CreateAllScenarioTestEnvFiles" >
+          DependsOnTargets="CreateAllScenarioTestEnvFiles" >
   </Target>
 
   <!--
diff --git a/src/coreclr/tests/CoreFX/CoreFX.issues.rsp b/src/coreclr/tests/CoreFX/CoreFX.issues.rsp
deleted file mode 100644
index c8ffa4edb77fd3..00000000000000
--- a/src/coreclr/tests/CoreFX/CoreFX.issues.rsp
+++ /dev/null
@@ -1,73 +0,0 @@
-# This is a "response file" used to pass fine-grained test exclusions to the
-# corefx xunit test wrapper scripts, RunTests.cmd or RunTests.sh. Lines here
-# should be in a format that xunit understands. Lines beginning with '#' are
-# comment lines and are ignored.
-#
-# The interesting xunit exclusion syntax is (extracted from help screen for
-# "dotnet.exe xunit.console.dll -?"):
-#
-# -------------------------------------------------------------------------
-# -nomethod "name"       : do not run a given test method (can be fully specified or use a wildcard;
-#                        : i.e., 'MyNamespace.MyClass.MyTestMethod' or '*.MyTestMethod')
-#                        : if specified more than once, acts as an AND operation
-# -noclass "name"        : do not run any methods in a given test class (should be fully
-#                        : specified; i.e., 'MyNamespace.MyClass')
-#                        : if specified more than once, acts as an AND operation
-# -nonamespace "name"    : do not run any methods in a given namespace (i.e.,
-#                        : 'MyNamespace.MySubNamespace')
-#                        : if specified more than once, acts as an AND operation
-# -------------------------------------------------------------------------
-#
-# Please list a GitHub issue for every exclusion.
-
--notrait category=OuterLoop
--notrait category=RequiresElevation
-
-# System.ComponentModel.Composition.Registration.Tests: test failures
--noclass System.ComponentModel.Composition.Registration.Tests.RegistrationBuilderAttributedOverrideUnitTests
-
-# Timeout on Linux/arm32
--nonamespace System.Data.SqlClient.Tests
-
-# Missing OOB dependencies (System.Utf8String.Experimental) - Disable UTF8 string tests
--noclass System.Net.Http.Tests.Utf8StringContentTests
--noclass System.Tests.Char8Tests
--noclass System.Tests.MemoryTests
--noclass System.Tests.ReflectionTests
--noclass System.Tests.Utf8ExtensionsTests
--noclass System.Tests.Utf8StringTests
--noclass System.Text.Tests.Utf8SpanTests
-
-# Missing OOB dependencies
--nonamespace System.Net.Quic.Tests
-
-# System.Net.Tests are known for their instability
--nonamespace System.Net.Http.Functional.Tests
--nonamespace System.Net.Sockets.Tests
-
--noclass System.Net.Tests.HttpListenerContextTests
--noclass System.Net.Tests.HttpListenerAuthenticationTests
--noclass System.Net.Tests.HttpListenerResponseTests
--noclass System.Net.Tests.HttpListenerResponseCookiesTests
-
--nomethod System.Net.Tests.HttpListenerResponseTests.Redirect_Invoke_SetsRedirectionProperties
--nomethod System.Net.Mail.Tests.SmtpClientTest.TestMailDeliveryAsync
-
--nomethod System.Net.Security.Tests.LoggingTest.EventSource_EventsRaisedAsExpected
--nomethod System.Net.Security.Tests.SslStreamSniTest.SslStream_NoSniFromClient_CallbackReturnsNull
-
-# System.Linq.Expressions.Tests and System.Linq.Expressions.Tests.ArrayBoundsTests.NewArrayBounds in particular are long-running:
-# https://github.com/dotnet/coreclr/issues/25269
--nomethod System.Linq.Expressions.Tests.ArrayBoundsTests.NewArrayBounds
--nonamespace System.Linq.Expressions.Tests
-
-# Timeout in System.Runtime.Serialization.Formatters.Tests.BinaryFormatterTests.SerializeHugeObjectGraphs: https://github.com/dotnet/coreclr/issues/20246
--nomethod System.Runtime.Serialization.Formatters.Tests.BinaryFormatterTests.SerializeHugeObjectGraphs
-
-# Timeout in System.Numerics.Tests.ToStringTest.RunRegionSpecificStandardFormatToStringTests
-# https://github.com/dotnet/coreclr/issues/22414
--nomethod System.Numerics.Tests.ToStringTest.RunRegionSpecificStandardFormatToStringTests
-
-# Timeout in System.Text.RegularExpressions.Tests.RegexMatchTests.Match_ExcessPrefix
-# https://github.com/dotnet/coreclr/issues/18912
--nomethod System.Text.RegularExpressions.Tests.RegexMatchTests.Match_ExcessPrefix
diff --git a/src/coreclr/tests/scripts/run-corefx-tests.bat b/src/coreclr/tests/scripts/run-corefx-tests.bat
index 2ff3a335a4a700..4848caa9700478 100644
--- a/src/coreclr/tests/scripts/run-corefx-tests.bat
+++ b/src/coreclr/tests/scripts/run-corefx-tests.bat
@@ -19,7 +19,6 @@ echo ^<test exclusion file^> -- Path to test exclusion file, e.g., C:\coreclr\te
 echo ^<architecture^>        -- Architecture to run, either ARM or ARM64. (We can't depend on PROCESSOR_ARCHITECTURE because
 echo                            the batch script might be invoked with an ARM64 CMD but we need to run ARM.)
 echo ^<exclusion rsp file^>  -- Path to test exclusion response file, passed to RunTests.cmd and then xunit, e.g.,
-echo                            C:\coreclr\tests\CoreFX\CoreFX.issues.rsp
 echo.
 echo The ^<test exclusion file^> is a file with a list of assemblies for which the
 echo tests should not be run. This allows excluding failing tests by excluding the
diff --git a/src/libraries/System.ComponentModel.Composition.Registration/tests/System/ComponentModel/Composition/Registration/RegistrationBuilderAttributedOverrideUnitTests.cs b/src/libraries/System.ComponentModel.Composition.Registration/tests/System/ComponentModel/Composition/Registration/RegistrationBuilderAttributedOverrideUnitTests.cs
index 3e72d281b13df1..778ad07b4f5659 100644
--- a/src/libraries/System.ComponentModel.Composition.Registration/tests/System/ComponentModel/Composition/Registration/RegistrationBuilderAttributedOverrideUnitTests.cs
+++ b/src/libraries/System.ComponentModel.Composition.Registration/tests/System/ComponentModel/Composition/Registration/RegistrationBuilderAttributedOverrideUnitTests.cs
@@ -11,6 +11,7 @@
 
 namespace System.ComponentModel.Composition.Registration.Tests
 {
+    [SkipOnCoreClr("Test failures on stress tests")]
     public class RegistrationBuilderAttributedOverrideUnitTests
     {
         public interface IContractA { }
diff --git a/src/libraries/System.Data.SqlClient/tests/FunctionalTests/AssemblyInfo.cs b/src/libraries/System.Data.SqlClient/tests/FunctionalTests/AssemblyInfo.cs
new file mode 100644
index 00000000000000..abe7673853fac8
--- /dev/null
+++ b/src/libraries/System.Data.SqlClient/tests/FunctionalTests/AssemblyInfo.cs
@@ -0,0 +1,7 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Xunit;
+
+[assembly: SkipOnCoreClr("Timeout in stress tests on Linux/arm32", TestPlatforms.Linux)]
diff --git a/src/libraries/System.Data.SqlClient/tests/FunctionalTests/System.Data.SqlClient.Tests.csproj b/src/libraries/System.Data.SqlClient/tests/FunctionalTests/System.Data.SqlClient.Tests.csproj
index a72c9b9b83eceb..4573bf83e3e5dd 100644
--- a/src/libraries/System.Data.SqlClient/tests/FunctionalTests/System.Data.SqlClient.Tests.csproj
+++ b/src/libraries/System.Data.SqlClient/tests/FunctionalTests/System.Data.SqlClient.Tests.csproj
@@ -4,6 +4,7 @@
     <Configurations>netcoreapp-Unix-Debug;netcoreapp-Unix-Release;netcoreapp-Windows_NT-Debug;netcoreapp-Windows_NT-Release;netfx-Windows_NT-Debug;netfx-Windows_NT-Release</Configurations>
   </PropertyGroup>
   <ItemGroup>
+    <Compile Include="AssemblyInfo.cs" />
     <Compile Include="AADAccessTokenTest.cs" />
     <Compile Include="CloneTests.cs" />
     <Compile Include="BaseProviderAsyncTest\BaseProviderAsyncTest.cs" />
diff --git a/src/libraries/System.Linq.Expressions/tests/AssemblyInfo.cs b/src/libraries/System.Linq.Expressions/tests/AssemblyInfo.cs
new file mode 100644
index 00000000000000..7860b6f42f2145
--- /dev/null
+++ b/src/libraries/System.Linq.Expressions/tests/AssemblyInfo.cs
@@ -0,0 +1,7 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Xunit;
+
+[assembly: SkipOnCoreClr("Long running tests: https://github.com/dotnet/coreclr/issues/25269", RuntimeStressTestModes.CheckedRuntime)]
diff --git a/src/libraries/System.Linq.Expressions/tests/System.Linq.Expressions.Tests.csproj b/src/libraries/System.Linq.Expressions/tests/System.Linq.Expressions.Tests.csproj
index c99c38eb7cf387..1dd4394f3da6d2 100644
--- a/src/libraries/System.Linq.Expressions/tests/System.Linq.Expressions.Tests.csproj
+++ b/src/libraries/System.Linq.Expressions/tests/System.Linq.Expressions.Tests.csproj
@@ -15,6 +15,7 @@
     <AssembliesBeingTested Include="Microsoft.CSharp" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="AssemblyInfo.cs" />
     <Compile Include="CompilerTests.cs" />
     <Compile Include="Array\ArrayAccessTests.cs" />
     <Compile Include="Array\ArrayArrayIndexTests.cs" />
diff --git a/src/libraries/System.Net.Http/tests/FunctionalTests/AssemblyInfo.cs b/src/libraries/System.Net.Http/tests/FunctionalTests/AssemblyInfo.cs
new file mode 100644
index 00000000000000..2addd5818288b9
--- /dev/null
+++ b/src/libraries/System.Net.Http/tests/FunctionalTests/AssemblyInfo.cs
@@ -0,0 +1,7 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Xunit;
+
+[assembly: SkipOnCoreClr("System.Net.Tests are inestable")]
diff --git a/src/libraries/System.Net.Http/tests/FunctionalTests/System.Net.Http.Functional.Tests.csproj b/src/libraries/System.Net.Http/tests/FunctionalTests/System.Net.Http.Functional.Tests.csproj
index a08d875236d761..c10bedc06b25f1 100644
--- a/src/libraries/System.Net.Http/tests/FunctionalTests/System.Net.Http.Functional.Tests.csproj
+++ b/src/libraries/System.Net.Http/tests/FunctionalTests/System.Net.Http.Functional.Tests.csproj
@@ -85,6 +85,7 @@
     <Compile Include="$(CommonTestPath)System\Threading\Tasks\TaskTimeoutExtensions.cs">
       <Link>Common\System\Threading\Tasks\TaskTimeoutExtensions.cs</Link>
     </Compile>
+    <Compile Include="AssemblyInfo.cs" />
     <Compile Include="ByteArrayContentTest.cs" />
     <Compile Include="ChannelBindingAwareContent.cs" />
     <Compile Include="DribbleStream.cs" />
diff --git a/src/libraries/System.Net.HttpListener/tests/HttpListenerAuthenticationTests.cs b/src/libraries/System.Net.HttpListener/tests/HttpListenerAuthenticationTests.cs
index 20795942a86b50..d2e8003c69335b 100644
--- a/src/libraries/System.Net.HttpListener/tests/HttpListenerAuthenticationTests.cs
+++ b/src/libraries/System.Net.HttpListener/tests/HttpListenerAuthenticationTests.cs
@@ -13,6 +13,7 @@
 
 namespace System.Net.Tests
 {
+    [SkipOnCoreClr("System.Net.Tests are inestable")]
     [ConditionalClass(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoServer))] // httpsys component missing in Nano.
     public class HttpListenerAuthenticationTests : IDisposable
     {
diff --git a/src/libraries/System.Net.HttpListener/tests/HttpListenerContextTests.cs b/src/libraries/System.Net.HttpListener/tests/HttpListenerContextTests.cs
index 137bc186a2b6c7..4a542440c675a1 100644
--- a/src/libraries/System.Net.HttpListener/tests/HttpListenerContextTests.cs
+++ b/src/libraries/System.Net.HttpListener/tests/HttpListenerContextTests.cs
@@ -14,6 +14,7 @@
 
 namespace System.Net.Tests
 {
+    [SkipOnCoreClr("System.Net.Tests are inestable")]
     [ConditionalClass(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoServer))] // httpsys component missing in Nano.
     public class HttpListenerContextTests : IDisposable
     {
diff --git a/src/libraries/System.Net.HttpListener/tests/HttpListenerResponseTests.Cookies.cs b/src/libraries/System.Net.HttpListener/tests/HttpListenerResponseTests.Cookies.cs
index 33855453473d41..4bcfddc7d1ca25 100644
--- a/src/libraries/System.Net.HttpListener/tests/HttpListenerResponseTests.Cookies.cs
+++ b/src/libraries/System.Net.HttpListener/tests/HttpListenerResponseTests.Cookies.cs
@@ -9,6 +9,7 @@
 
 namespace System.Net.Tests
 {
+    [SkipOnCoreClr("System.Net.Tests are inestable")]
     [ConditionalClass(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoServer))] // httpsys component missing in Nano.
     public class HttpListenerResponseCookiesTests : HttpListenerResponseTestBase
     {
diff --git a/src/libraries/System.Net.HttpListener/tests/HttpListenerResponseTests.cs b/src/libraries/System.Net.HttpListener/tests/HttpListenerResponseTests.cs
index 83332ffe22dfa4..a77eb079ef123c 100644
--- a/src/libraries/System.Net.HttpListener/tests/HttpListenerResponseTests.cs
+++ b/src/libraries/System.Net.HttpListener/tests/HttpListenerResponseTests.cs
@@ -57,6 +57,7 @@ protected async Task<HttpListenerResponse> GetResponse(string httpVersion = "1.1
         }
     }
 
+    [SkipOnCoreClr("System.Net.Tests are inestable")]
     [ConditionalClass(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoServer))] // httpsys component missing in Nano.
     public class HttpListenerResponseTests : HttpListenerResponseTestBase
     {
@@ -112,6 +113,7 @@ public async Task CopyFrom_NullTemplateResponse_ThrowsNullReferenceException()
         [InlineData(" \r \t \n", 123)]
         [InlineData("http://microsoft.com", 155)]
         [InlineData("  http://microsoft.com  ", 155)]
+        [SkipOnCoreClr("System.Net.Tests are inestable")]
         public async Task Redirect_Invoke_SetsRedirectionProperties(string url, int expectedNumberOfBytes)
         {
             string expectedUrl = url?.Trim() ?? "";
diff --git a/src/libraries/System.Net.Mail/tests/Functional/SmtpClientTest.cs b/src/libraries/System.Net.Mail/tests/Functional/SmtpClientTest.cs
index dd72e7e7836eda..a2e1cab84bd812 100644
--- a/src/libraries/System.Net.Mail/tests/Functional/SmtpClientTest.cs
+++ b/src/libraries/System.Net.Mail/tests/Functional/SmtpClientTest.cs
@@ -347,6 +347,7 @@ public void TestZeroTimeout()
         [InlineData("howdydoo")]
         [InlineData("")]
         [InlineData(null)]
+        [SkipOnCoreClr("System.Net.Tests are inestable")]
         public async Task TestMailDeliveryAsync(string body)
         {
             SmtpServer server = new SmtpServer();
diff --git a/src/libraries/System.Net.Requests/tests/LoggingTest.cs b/src/libraries/System.Net.Requests/tests/LoggingTest.cs
index f0b52db6f8ce44..4994c18d7020dc 100644
--- a/src/libraries/System.Net.Requests/tests/LoggingTest.cs
+++ b/src/libraries/System.Net.Requests/tests/LoggingTest.cs
@@ -11,6 +11,7 @@ public class LoggingTest
     {
         [Fact]
         [SkipOnTargetFramework(TargetFrameworkMonikers.Mono, "NetEventSource is only part of .NET Core.")]
+        [SkipOnCoreClr("System.Net.Tests are inestable")]
         public void EventSource_ExistsWithCorrectId()
         {
             Type esType = typeof(WebRequest).Assembly.GetType("System.Net.NetEventSource", throwOnError: true, ignoreCase: false);
diff --git a/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamSniTest.cs b/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamSniTest.cs
index 07696a1decd269..bebc608f2b6389 100644
--- a/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamSniTest.cs
+++ b/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamSniTest.cs
@@ -137,6 +137,7 @@ public async Task SslStream_ServerCallbackNotSet_UsesLocalCertificateSelection(s
         }
 
         [Fact]
+        [SkipOnCoreClr("System.Net.Tests are inestable")]
         public async Task SslStream_NoSniFromClient_CallbackReturnsNull()
         {
             await WithVirtualConnection(async (server, client) =>
diff --git a/src/libraries/System.Net.Sockets/tests/FunctionalTests/AssemblyInfo.cs b/src/libraries/System.Net.Sockets/tests/FunctionalTests/AssemblyInfo.cs
new file mode 100644
index 00000000000000..2addd5818288b9
--- /dev/null
+++ b/src/libraries/System.Net.Sockets/tests/FunctionalTests/AssemblyInfo.cs
@@ -0,0 +1,7 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Xunit;
+
+[assembly: SkipOnCoreClr("System.Net.Tests are inestable")]
diff --git a/src/libraries/System.Net.Sockets/tests/FunctionalTests/System.Net.Sockets.Tests.csproj b/src/libraries/System.Net.Sockets/tests/FunctionalTests/System.Net.Sockets.Tests.csproj
index 8cf503a718443c..2d905398a9b68d 100644
--- a/src/libraries/System.Net.Sockets/tests/FunctionalTests/System.Net.Sockets.Tests.csproj
+++ b/src/libraries/System.Net.Sockets/tests/FunctionalTests/System.Net.Sockets.Tests.csproj
@@ -7,6 +7,7 @@
   <ItemGroup>
     <Compile Include="Accept.cs" />
     <Compile Include="AgnosticListenerTest.cs" />
+    <Compile Include="AssemblyInfo.cs" />
     <Compile Include="ArgumentValidationTests.cs" />
     <Compile Include="DisconnectTest.cs" />
     <Compile Include="HandleTest.cs" />
diff --git a/src/libraries/System.Runtime.Numerics/tests/BigInteger/BigIntegerToStringTests.cs b/src/libraries/System.Runtime.Numerics/tests/BigInteger/BigIntegerToStringTests.cs
index 2be886c05194c3..2e99d07489234d 100644
--- a/src/libraries/System.Runtime.Numerics/tests/BigInteger/BigIntegerToStringTests.cs
+++ b/src/libraries/System.Runtime.Numerics/tests/BigInteger/BigIntegerToStringTests.cs
@@ -188,6 +188,7 @@ public static void RunStandardFormatToStringTests()
         }
 
         [Fact]
+        [SkipOnCoreClr("Long running tests: https://github.com/dotnet/coreclr/issues/22414", TestPlatforms.Linux, RuntimeStressTestModes.CheckedRuntime)]
         public static void RunRegionSpecificStandardFormatToStringTests()
         {
             CultureInfo[] cultures = new CultureInfo[] { new CultureInfo("en-US"), new CultureInfo("en-GB"), new CultureInfo("fr-CA"),
diff --git a/src/libraries/System.Runtime.Serialization.Formatters/tests/BinaryFormatterTests.cs b/src/libraries/System.Runtime.Serialization.Formatters/tests/BinaryFormatterTests.cs
index 5232c1a2764001..869a032e97ea02 100644
--- a/src/libraries/System.Runtime.Serialization.Formatters/tests/BinaryFormatterTests.cs
+++ b/src/libraries/System.Runtime.Serialization.Formatters/tests/BinaryFormatterTests.cs
@@ -23,6 +23,7 @@ public partial class BinaryFormatterTests : FileCleanupTestBase
     {
         // On 32-bit we can't test these high inputs as they cause OutOfMemoryExceptions.
         [ConditionalTheory(typeof(Environment), nameof(Environment.Is64BitProcess))]
+        [SkipOnCoreClr("Long running tests: https://github.com/dotnet/coreclr/issues/20246", RuntimeStressTestModes.CheckedRuntime)]
         [InlineData(2 * 6_584_983 - 2)] // previous limit
         [InlineData(2 * 7_199_369 - 2)] // last pre-computed prime number
         public void SerializeHugeObjectGraphs(int limit)
diff --git a/src/libraries/System.Text.RegularExpressions/tests/Regex.Match.Tests.cs b/src/libraries/System.Text.RegularExpressions/tests/Regex.Match.Tests.cs
index f4135d31d0a368..a701caf0a80ccb 100644
--- a/src/libraries/System.Text.RegularExpressions/tests/Regex.Match.Tests.cs
+++ b/src/libraries/System.Text.RegularExpressions/tests/Regex.Match.Tests.cs
@@ -807,6 +807,7 @@ public void Match_SpecialUnicodeCharacters_Invariant()
         }
 
         [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotArmProcess))] // times out on ARM
+        [SkipOnCoreClr("Long running tests: https://github.com/dotnet/coreclr/issues/18912", RuntimeStressTestModes.JitMinOpts)]
         public void Match_ExcessPrefix()
         {
             RemoteExecutor.Invoke(() =>
diff --git a/src/libraries/System.Utf8String.Experimental/tests/System.Utf8String.Experimental.Tests.csproj b/src/libraries/System.Utf8String.Experimental/tests/System.Utf8String.Experimental.Tests.csproj
index e21b2a32c79293..46fcb49f455a9e 100644
--- a/src/libraries/System.Utf8String.Experimental/tests/System.Utf8String.Experimental.Tests.csproj
+++ b/src/libraries/System.Utf8String.Experimental/tests/System.Utf8String.Experimental.Tests.csproj
@@ -12,6 +12,7 @@
     <IncludeUtf8StringAsmRef>true</IncludeUtf8StringAsmRef>
   </PropertyGroup>
   <ItemGroup Condition="'$(IsPrerelease)' != 'false'">
+    <Compile Include="AssemblyInfo.cs" />
     <Compile Include="System\Char8Tests.cs" />
     <Compile Include="System\MemoryTests.cs" />
     <Compile Include="System\Net\Http\Utf8StringContentTests.cs" />

From a6cb2a63ddd258b15a3bfdff719f344d6311a515 Mon Sep 17 00:00:00 2001
From: Santiago Fernandez Madero <safern@microsoft.com>
Date: Tue, 19 Nov 2019 13:08:05 -0800
Subject: [PATCH 2/2] Address PR Feedback

---
 eng/helixcorefxtests.proj                     | 28 ++++++-
 src/coreclr/tests/CoreFX/CoreFX.issues.rsp    | 73 +++++++++++++++++++
 .../tests/scripts/run-corefx-tests.bat        |  1 +
 .../tests/FunctionalTests/AssemblyInfo.cs     |  2 +-
 .../tests/Functional/SmtpClientTest.cs        |  2 +-
 .../tests/FunctionalTests/SslStreamSniTest.cs |  2 +-
 .../tests/FunctionalTests/AssemblyInfo.cs     |  2 +-
 ...ystem.Utf8String.Experimental.Tests.csproj |  1 -
 8 files changed, 103 insertions(+), 8 deletions(-)
 create mode 100644 src/coreclr/tests/CoreFX/CoreFX.issues.rsp

diff --git a/eng/helixcorefxtests.proj b/eng/helixcorefxtests.proj
index 5eb26c3a77c31d..7b14fb1d0ca70a 100644
--- a/eng/helixcorefxtests.proj
+++ b/eng/helixcorefxtests.proj
@@ -89,6 +89,15 @@
     <TestWorkingDir Condition="'$(__TestWorkingDir)'==''">$(ProjectDir)artifacts\tests\coreclr\$(__BuildOS).$(__BuildArch).$(__BuildType)\</TestWorkingDir>
     <TargetsWindows Condition="'$(__BuildOS)' == 'Windows_NT'">true</TargetsWindows>
 
+    <!-- We have a single, universal exclusion file for all CoreFX test exclusions, for CoreFX tests run
+         in the CoreCLR repo. Note in particular that we don't have a unique exclusion file for each test
+         assembly, or for each processor architecture, or for each operating system, or for each stress
+         scenario type. This means that we generally exclude more tests than is strictly necessary for
+         any particular job. However, this mechanism is super simple; we don't need to manage fine-
+         grained exclusions; and there really shouldn't be any (or many) exclusions anyway.
+    -->
+    <CoreFXTestExclusionFile>$(ProjectDir)src\coreclr\tests\CoreFX\CoreFX.issues.rsp</CoreFXTestExclusionFile>
+
     <TestHostRootPath>$(TestWorkingDir)testhost\</TestHostRootPath>
     <TestArchiveRuntimeRoot>$(TestWorkingDir)helix\</TestArchiveRuntimeRoot>
 
@@ -109,10 +118,23 @@
       The arguments passed in to the run script in are the runtime directory (-r) and the exclusion response file
       passed to xunit (rsp-file).
     -->
-    <HelixCommand Condition="'$(TargetsWindows)' == 'true'">call RunTests.cmd -r %HELIX_CORRELATION_PAYLOAD%</HelixCommand>
-    <HelixCommand Condition="'$(TargetsWindows)' != 'true'">./RunTests.sh -r $HELIX_CORRELATION_PAYLOAD</HelixCommand>
+    <HelixCommand Condition="'$(TargetsWindows)' == 'true'">call RunTests.cmd -r %HELIX_CORRELATION_PAYLOAD% --rsp-file %HELIX_CORRELATION_PAYLOAD%\CoreFX.issues.rsp </HelixCommand>
+    <HelixCommand Condition="'$(TargetsWindows)' != 'true'">./RunTests.sh -r $HELIX_CORRELATION_PAYLOAD --rsp-file $HELIX_CORRELATION_PAYLOAD/CoreFX.issues.rsp </HelixCommand>
   </PropertyGroup>
 
+  <!--
+    Copy the CoreFX test exclusion response file (which is passed to the xunit console runner) to someplace we can
+    access it during the test run. We currently just copy it to the root of the generated .NET Core testhost
+    directory, which we use as the root of the correlation directory.
+  -->
+  <Target Name="CopyRSPFile">
+    <Copy
+      SourceFiles="$(CoreFXTestExclusionFile)"
+      DestinationFolder="$(TestHostRootPath)" />
+
+    <Message Importance="High" Text="Copied $(CoreFXTestExclusionFile) into $(TestHostRootPath)" />
+  </Target>
+
   <PropertyGroup>
     <!-- Set the name of the scenario file. Note that this is only used in invocations where $(Scenario) is set. -->
     <TestEnvFileName Condition=" '$(TargetsWindows)' == 'true' ">SetStressModes_$(Scenario).cmd</TestEnvFileName>
@@ -156,7 +178,7 @@
     This just causes its dependent targets to be run.
   -->
   <Target Name="PrepareCorrelationPayloadDirectory"
-          DependsOnTargets="CreateAllScenarioTestEnvFiles" >
+          DependsOnTargets="CopyRSPFile;CreateAllScenarioTestEnvFiles" >
   </Target>
 
   <!--
diff --git a/src/coreclr/tests/CoreFX/CoreFX.issues.rsp b/src/coreclr/tests/CoreFX/CoreFX.issues.rsp
new file mode 100644
index 00000000000000..c8ffa4edb77fd3
--- /dev/null
+++ b/src/coreclr/tests/CoreFX/CoreFX.issues.rsp
@@ -0,0 +1,73 @@
+# This is a "response file" used to pass fine-grained test exclusions to the
+# corefx xunit test wrapper scripts, RunTests.cmd or RunTests.sh. Lines here
+# should be in a format that xunit understands. Lines beginning with '#' are
+# comment lines and are ignored.
+#
+# The interesting xunit exclusion syntax is (extracted from help screen for
+# "dotnet.exe xunit.console.dll -?"):
+#
+# -------------------------------------------------------------------------
+# -nomethod "name"       : do not run a given test method (can be fully specified or use a wildcard;
+#                        : i.e., 'MyNamespace.MyClass.MyTestMethod' or '*.MyTestMethod')
+#                        : if specified more than once, acts as an AND operation
+# -noclass "name"        : do not run any methods in a given test class (should be fully
+#                        : specified; i.e., 'MyNamespace.MyClass')
+#                        : if specified more than once, acts as an AND operation
+# -nonamespace "name"    : do not run any methods in a given namespace (i.e.,
+#                        : 'MyNamespace.MySubNamespace')
+#                        : if specified more than once, acts as an AND operation
+# -------------------------------------------------------------------------
+#
+# Please list a GitHub issue for every exclusion.
+
+-notrait category=OuterLoop
+-notrait category=RequiresElevation
+
+# System.ComponentModel.Composition.Registration.Tests: test failures
+-noclass System.ComponentModel.Composition.Registration.Tests.RegistrationBuilderAttributedOverrideUnitTests
+
+# Timeout on Linux/arm32
+-nonamespace System.Data.SqlClient.Tests
+
+# Missing OOB dependencies (System.Utf8String.Experimental) - Disable UTF8 string tests
+-noclass System.Net.Http.Tests.Utf8StringContentTests
+-noclass System.Tests.Char8Tests
+-noclass System.Tests.MemoryTests
+-noclass System.Tests.ReflectionTests
+-noclass System.Tests.Utf8ExtensionsTests
+-noclass System.Tests.Utf8StringTests
+-noclass System.Text.Tests.Utf8SpanTests
+
+# Missing OOB dependencies
+-nonamespace System.Net.Quic.Tests
+
+# System.Net.Tests are known for their instability
+-nonamespace System.Net.Http.Functional.Tests
+-nonamespace System.Net.Sockets.Tests
+
+-noclass System.Net.Tests.HttpListenerContextTests
+-noclass System.Net.Tests.HttpListenerAuthenticationTests
+-noclass System.Net.Tests.HttpListenerResponseTests
+-noclass System.Net.Tests.HttpListenerResponseCookiesTests
+
+-nomethod System.Net.Tests.HttpListenerResponseTests.Redirect_Invoke_SetsRedirectionProperties
+-nomethod System.Net.Mail.Tests.SmtpClientTest.TestMailDeliveryAsync
+
+-nomethod System.Net.Security.Tests.LoggingTest.EventSource_EventsRaisedAsExpected
+-nomethod System.Net.Security.Tests.SslStreamSniTest.SslStream_NoSniFromClient_CallbackReturnsNull
+
+# System.Linq.Expressions.Tests and System.Linq.Expressions.Tests.ArrayBoundsTests.NewArrayBounds in particular are long-running:
+# https://github.com/dotnet/coreclr/issues/25269
+-nomethod System.Linq.Expressions.Tests.ArrayBoundsTests.NewArrayBounds
+-nonamespace System.Linq.Expressions.Tests
+
+# Timeout in System.Runtime.Serialization.Formatters.Tests.BinaryFormatterTests.SerializeHugeObjectGraphs: https://github.com/dotnet/coreclr/issues/20246
+-nomethod System.Runtime.Serialization.Formatters.Tests.BinaryFormatterTests.SerializeHugeObjectGraphs
+
+# Timeout in System.Numerics.Tests.ToStringTest.RunRegionSpecificStandardFormatToStringTests
+# https://github.com/dotnet/coreclr/issues/22414
+-nomethod System.Numerics.Tests.ToStringTest.RunRegionSpecificStandardFormatToStringTests
+
+# Timeout in System.Text.RegularExpressions.Tests.RegexMatchTests.Match_ExcessPrefix
+# https://github.com/dotnet/coreclr/issues/18912
+-nomethod System.Text.RegularExpressions.Tests.RegexMatchTests.Match_ExcessPrefix
diff --git a/src/coreclr/tests/scripts/run-corefx-tests.bat b/src/coreclr/tests/scripts/run-corefx-tests.bat
index 4848caa9700478..2ff3a335a4a700 100644
--- a/src/coreclr/tests/scripts/run-corefx-tests.bat
+++ b/src/coreclr/tests/scripts/run-corefx-tests.bat
@@ -19,6 +19,7 @@ echo ^<test exclusion file^> -- Path to test exclusion file, e.g., C:\coreclr\te
 echo ^<architecture^>        -- Architecture to run, either ARM or ARM64. (We can't depend on PROCESSOR_ARCHITECTURE because
 echo                            the batch script might be invoked with an ARM64 CMD but we need to run ARM.)
 echo ^<exclusion rsp file^>  -- Path to test exclusion response file, passed to RunTests.cmd and then xunit, e.g.,
+echo                            C:\coreclr\tests\CoreFX\CoreFX.issues.rsp
 echo.
 echo The ^<test exclusion file^> is a file with a list of assemblies for which the
 echo tests should not be run. This allows excluding failing tests by excluding the
diff --git a/src/libraries/System.Net.Http/tests/FunctionalTests/AssemblyInfo.cs b/src/libraries/System.Net.Http/tests/FunctionalTests/AssemblyInfo.cs
index 2addd5818288b9..282c71ecae151c 100644
--- a/src/libraries/System.Net.Http/tests/FunctionalTests/AssemblyInfo.cs
+++ b/src/libraries/System.Net.Http/tests/FunctionalTests/AssemblyInfo.cs
@@ -4,4 +4,4 @@
 
 using Xunit;
 
-[assembly: SkipOnCoreClr("System.Net.Tests are inestable")]
+[assembly: SkipOnCoreClr("System.Net.Tests are flaky and/or long running: https://github.com/dotnet/runtime/issues/131")]
diff --git a/src/libraries/System.Net.Mail/tests/Functional/SmtpClientTest.cs b/src/libraries/System.Net.Mail/tests/Functional/SmtpClientTest.cs
index a2e1cab84bd812..53982f22b0f7e7 100644
--- a/src/libraries/System.Net.Mail/tests/Functional/SmtpClientTest.cs
+++ b/src/libraries/System.Net.Mail/tests/Functional/SmtpClientTest.cs
@@ -347,7 +347,7 @@ public void TestZeroTimeout()
         [InlineData("howdydoo")]
         [InlineData("")]
         [InlineData(null)]
-        [SkipOnCoreClr("System.Net.Tests are inestable")]
+        [SkipOnCoreClr("System.Net.Tests are flaky and/or long running: https://github.com/dotnet/runtime/issues/131")]
         public async Task TestMailDeliveryAsync(string body)
         {
             SmtpServer server = new SmtpServer();
diff --git a/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamSniTest.cs b/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamSniTest.cs
index bebc608f2b6389..f05bdc199c00ed 100644
--- a/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamSniTest.cs
+++ b/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamSniTest.cs
@@ -137,7 +137,7 @@ public async Task SslStream_ServerCallbackNotSet_UsesLocalCertificateSelection(s
         }
 
         [Fact]
-        [SkipOnCoreClr("System.Net.Tests are inestable")]
+        [SkipOnCoreClr("System.Net.Tests are flaky and/or long running: https://github.com/dotnet/runtime/issues/131")]
         public async Task SslStream_NoSniFromClient_CallbackReturnsNull()
         {
             await WithVirtualConnection(async (server, client) =>
diff --git a/src/libraries/System.Net.Sockets/tests/FunctionalTests/AssemblyInfo.cs b/src/libraries/System.Net.Sockets/tests/FunctionalTests/AssemblyInfo.cs
index 2addd5818288b9..282c71ecae151c 100644
--- a/src/libraries/System.Net.Sockets/tests/FunctionalTests/AssemblyInfo.cs
+++ b/src/libraries/System.Net.Sockets/tests/FunctionalTests/AssemblyInfo.cs
@@ -4,4 +4,4 @@
 
 using Xunit;
 
-[assembly: SkipOnCoreClr("System.Net.Tests are inestable")]
+[assembly: SkipOnCoreClr("System.Net.Tests are flaky and/or long running: https://github.com/dotnet/runtime/issues/131")]
diff --git a/src/libraries/System.Utf8String.Experimental/tests/System.Utf8String.Experimental.Tests.csproj b/src/libraries/System.Utf8String.Experimental/tests/System.Utf8String.Experimental.Tests.csproj
index 46fcb49f455a9e..e21b2a32c79293 100644
--- a/src/libraries/System.Utf8String.Experimental/tests/System.Utf8String.Experimental.Tests.csproj
+++ b/src/libraries/System.Utf8String.Experimental/tests/System.Utf8String.Experimental.Tests.csproj
@@ -12,7 +12,6 @@
     <IncludeUtf8StringAsmRef>true</IncludeUtf8StringAsmRef>
   </PropertyGroup>
   <ItemGroup Condition="'$(IsPrerelease)' != 'false'">
-    <Compile Include="AssemblyInfo.cs" />
     <Compile Include="System\Char8Tests.cs" />
     <Compile Include="System\MemoryTests.cs" />
     <Compile Include="System\Net\Http\Utf8StringContentTests.cs" />