Skip to content

Disable tests on CoreCLR using new attribute #118

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
safern wants to merge 2 commits into from
Closed

Disable tests on CoreCLR using new attribute #118

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ce89a9a
Disable tests on CoreCLR using new attribute
safern Nov 18, 2019
a6cb2a6
Address PR Feedback
safern Nov 19, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions NuGet.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
<add key="dotnet5-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json" />
<add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
<add key="dotnet-coreclr" value="https://dotnetfeed.blob.core.windows.net/dotnet-coreclr/index.json" />
<add key="safern-test" value="https://www.myget.org/F/safern-test/api/v3/index.json" />
Copy link
Contributor

@BruceForstall BruceForstall Nov 19, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I presume this will get removed before you merge?

Copy link
Member Author

@safern safern Nov 19, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. I’m just waiting for arcade to propagate.

</packageSources>
<disabledPackageSources>
<clear />
Expand Down
2 changes: 1 addition & 1 deletion eng/Versions.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
<MicrosoftDotNetCodeAnalysisVersion>5.0.0-beta.19567.2</MicrosoftDotNetCodeAnalysisVersion>
<MicrosoftDotNetGenAPIVersion>5.0.0-beta.19567.2</MicrosoftDotNetGenAPIVersion>
<MicrosoftDotNetGenFacadesVersion>5.0.0-beta.19567.2</MicrosoftDotNetGenFacadesVersion>
<MicrosoftDotNetXUnitExtensionsVersion>5.0.0-beta.19567.2</MicrosoftDotNetXUnitExtensionsVersion>
<MicrosoftDotNetXUnitExtensionsVersion>5.0.0-dev</MicrosoftDotNetXUnitExtensionsVersion>
<MicrosoftDotNetXUnitConsoleRunnerVersion>2.5.1-beta.19567.2</MicrosoftDotNetXUnitConsoleRunnerVersion>
<MicrosoftDotNetBuildTasksPackagingVersion>5.0.0-beta.19567.2</MicrosoftDotNetBuildTasksPackagingVersion>
<MicrosoftDotNetRemoteExecutorVersion>5.0.0-beta.19567.2</MicrosoftDotNetRemoteExecutorVersion>
Expand Down
1 change: 1 addition & 0 deletions ...ComponentModel/Composition/Registration/RegistrationBuilderAttributedOverrideUnitTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

namespace System.ComponentModel.Composition.Registration.Tests
{
[SkipOnCoreClr("Test failures on stress tests")]
public class RegistrationBuilderAttributedOverrideUnitTests
{
public interface IContractA { }
Expand Down
7 changes: 7 additions & 0 deletions src/libraries/System.Data.SqlClient/tests/FunctionalTests/AssemblyInfo.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using Xunit;

[assembly: SkipOnCoreClr("Timeout in stress tests on Linux/arm32", TestPlatforms.Linux)]
1 change: 1 addition & 0 deletions src/libraries/System.Data.SqlClient/tests/FunctionalTests/System.Data.SqlClient.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
<Configurations>netcoreapp-Unix-Debug;netcoreapp-Unix-Release;netcoreapp-Windows_NT-Debug;netcoreapp-Windows_NT-Release;netfx-Windows_NT-Debug;netfx-Windows_NT-Release</Configurations>
</PropertyGroup>
<ItemGroup>
<Compile Include="AssemblyInfo.cs" />
<Compile Include="AADAccessTokenTest.cs" />
<Compile Include="CloneTests.cs" />
<Compile Include="BaseProviderAsyncTest\BaseProviderAsyncTest.cs" />
Expand Down
7 changes: 7 additions & 0 deletions src/libraries/System.Linq.Expressions/tests/AssemblyInfo.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using Xunit;

[assembly: SkipOnCoreClr("Long running tests: https://github.com/dotnet/coreclr/issues/25269", RuntimeStressTestModes.CheckedRuntime)]
1 change: 1 addition & 0 deletions src/libraries/System.Linq.Expressions/tests/System.Linq.Expressions.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
<AssembliesBeingTested Include="Microsoft.CSharp" />
</ItemGroup>
<ItemGroup>
<Compile Include="AssemblyInfo.cs" />
<Compile Include="CompilerTests.cs" />
<Compile Include="Array\ArrayAccessTests.cs" />
<Compile Include="Array\ArrayArrayIndexTests.cs" />
Expand Down
7 changes: 7 additions & 0 deletions src/libraries/System.Net.Http/tests/FunctionalTests/AssemblyInfo.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using Xunit;

[assembly: SkipOnCoreClr("System.Net.Tests are flaky and/or long running: https://github.com/dotnet/runtime/issues/131")]
1 change: 1 addition & 0 deletions src/libraries/System.Net.Http/tests/FunctionalTests/System.Net.Http.Functional.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@
<Compile Include="$(CommonTestPath)System\Threading\Tasks\TaskTimeoutExtensions.cs">
<Link>Common\System\Threading\Tasks\TaskTimeoutExtensions.cs</Link>
</Compile>
<Compile Include="AssemblyInfo.cs" />
<Compile Include="ByteArrayContentTest.cs" />
<Compile Include="ChannelBindingAwareContent.cs" />
<Compile Include="DribbleStream.cs" />
Expand Down
1 change: 1 addition & 0 deletions src/libraries/System.Net.HttpListener/tests/HttpListenerAuthenticationTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

namespace System.Net.Tests
{
[SkipOnCoreClr("System.Net.Tests are inestable")]
[ConditionalClass(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoServer))] // httpsys component missing in Nano.
public class HttpListenerAuthenticationTests : IDisposable
{
Expand Down
1 change: 1 addition & 0 deletions src/libraries/System.Net.HttpListener/tests/HttpListenerContextTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

namespace System.Net.Tests
{
[SkipOnCoreClr("System.Net.Tests are inestable")]
[ConditionalClass(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoServer))] // httpsys component missing in Nano.
public class HttpListenerContextTests : IDisposable
{
Expand Down
1 change: 1 addition & 0 deletions src/libraries/System.Net.HttpListener/tests/HttpListenerResponseTests.Cookies.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

namespace System.Net.Tests
{
[SkipOnCoreClr("System.Net.Tests are inestable")]
[ConditionalClass(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoServer))] // httpsys component missing in Nano.
public class HttpListenerResponseCookiesTests : HttpListenerResponseTestBase
{
Expand Down
2 changes: 2 additions & 0 deletions src/libraries/System.Net.HttpListener/tests/HttpListenerResponseTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ protected async Task<HttpListenerResponse> GetResponse(string httpVersion = "1.1
}
}

[SkipOnCoreClr("System.Net.Tests are inestable")]
[ConditionalClass(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoServer))] // httpsys component missing in Nano.
public class HttpListenerResponseTests : HttpListenerResponseTestBase
{
Expand Down Expand Up @@ -112,6 +113,7 @@ public async Task CopyFrom_NullTemplateResponse_ThrowsNullReferenceException()
[InlineData(" \r \t \n", 123)]
[InlineData("http://microsoft.com", 155)]
[InlineData(" http://microsoft.com ", 155)]
[SkipOnCoreClr("System.Net.Tests are inestable")]
public async Task Redirect_Invoke_SetsRedirectionProperties(string url, int expectedNumberOfBytes)
{
string expectedUrl = url?.Trim() ?? "";
Expand Down
1 change: 1 addition & 0 deletions src/libraries/System.Net.Mail/tests/Functional/SmtpClientTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,7 @@ public void TestZeroTimeout()
[InlineData("howdydoo")]
[InlineData("")]
[InlineData(null)]
[SkipOnCoreClr("System.Net.Tests are flaky and/or long running: https://github.com/dotnet/runtime/issues/131")]
public async Task TestMailDeliveryAsync(string body)
{
SmtpServer server = new SmtpServer();
Expand Down
1 change: 1 addition & 0 deletions src/libraries/System.Net.Requests/tests/LoggingTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ public class LoggingTest
{
[Fact]
[SkipOnTargetFramework(TargetFrameworkMonikers.Mono, "NetEventSource is only part of .NET Core.")]
[SkipOnCoreClr("System.Net.Tests are inestable")]
public void EventSource_ExistsWithCorrectId()
{
Type esType = typeof(WebRequest).Assembly.GetType("System.Net.NetEventSource", throwOnError: true, ignoreCase: false);
Expand Down
1 change: 1 addition & 0 deletions src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamSniTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ public async Task SslStream_ServerCallbackNotSet_UsesLocalCertificateSelection(s
}

[Fact]
[SkipOnCoreClr("System.Net.Tests are flaky and/or long running: https://github.com/dotnet/runtime/issues/131")]
public async Task SslStream_NoSniFromClient_CallbackReturnsNull()
{
await WithVirtualConnection(async (server, client) =>
Expand Down
7 changes: 7 additions & 0 deletions src/libraries/System.Net.Sockets/tests/FunctionalTests/AssemblyInfo.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using Xunit;

[assembly: SkipOnCoreClr("System.Net.Tests are flaky and/or long running: https://github.com/dotnet/runtime/issues/131")]
Copy link
Contributor

@marek-safar marek-safar Nov 24, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If the tests are flaky why do you skip them on CoreCLR only?

Copy link
Member

@jkoritzinsky jkoritzinsky Nov 24, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If I remember correctly, they're flaky only on Checked CoreCLR because they have checks based on timeouts. The performance hits of running on a Checked CoreCLR leads to some of the timeout tests failing.

Copy link
Member

@stephentoub stephentoub Nov 24, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's not clear from the naming, but this is only skipping these tests in coreclr's stress test modes (including checked builds). We should a) fix the attribute to require that to be expressed such that no parameters means to always skip on coreclr, and b) we should fix the tests to be runnable in these cases, as well.

Copy link
Member Author

@safern safern Nov 25, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

SkipOnCoreClrStressTest sounds like a better naming?

Copy link
Member

@stephentoub stephentoub Nov 25, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

SkipOnCoreClrStressTest sounds like a better naming?

I think there are two options:

  1. Make the attribute specific to stress testing, in which case a name like you suggest would be fine.
  2. Actually make the attribute do what it's name says, which is to skip on coreclr. Then if you only wanted to skip on coreclr in a specific situation, you would pass the argument to say so, e.g. [SkipOnCoreClr] would always skip the test on coreclr, whereas [SkipOnCoreClr(CoreClrTestMode.Stress)] would skip it in stress scenarios, [SkipOnCoreClr(CoreClrTestMode.Checked | CoreClrTestMode.GCStress)] would skip only if using a checked build or running GC stress, etc.

Copy link
Member Author

@safern safern Nov 25, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, makes sense. I'll work later on that.

1 change: 1 addition & 0 deletions src/libraries/System.Net.Sockets/tests/FunctionalTests/System.Net.Sockets.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
<ItemGroup>
<Compile Include="Accept.cs" />
<Compile Include="AgnosticListenerTest.cs" />
<Compile Include="AssemblyInfo.cs" />
<Compile Include="ArgumentValidationTests.cs" />
<Compile Include="DisconnectTest.cs" />
<Compile Include="HandleTest.cs" />
Expand Down
1 change: 1 addition & 0 deletions src/libraries/System.Runtime.Numerics/tests/BigInteger/BigIntegerToStringTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,7 @@ public static void RunStandardFormatToStringTests()
}

[Fact]
[SkipOnCoreClr("Long running tests: https://github.com/dotnet/coreclr/issues/22414", TestPlatforms.Linux, RuntimeStressTestModes.CheckedRuntime)]
public static void RunRegionSpecificStandardFormatToStringTests()
{
CultureInfo[] cultures = new CultureInfo[] { new CultureInfo("en-US"), new CultureInfo("en-GB"), new CultureInfo("fr-CA"),
Expand Down
1 change: 1 addition & 0 deletions src/libraries/System.Runtime.Serialization.Formatters/tests/BinaryFormatterTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ public partial class BinaryFormatterTests : FileCleanupTestBase
{
// On 32-bit we can't test these high inputs as they cause OutOfMemoryExceptions.
[ConditionalTheory(typeof(Environment), nameof(Environment.Is64BitProcess))]
[SkipOnCoreClr("Long running tests: https://github.com/dotnet/coreclr/issues/20246", RuntimeStressTestModes.CheckedRuntime)]
[InlineData(2 * 6_584_983 - 2)] // previous limit
[InlineData(2 * 7_199_369 - 2)] // last pre-computed prime number
public void SerializeHugeObjectGraphs(int limit)
Expand Down
1 change: 1 addition & 0 deletions src/libraries/System.Text.RegularExpressions/tests/Regex.Match.Tests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -807,6 +807,7 @@ public void Match_SpecialUnicodeCharacters_Invariant()
}

[ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotArmProcess))] // times out on ARM
[SkipOnCoreClr("Long running tests: https://github.com/dotnet/coreclr/issues/18912", RuntimeStressTestModes.JitMinOpts)]
public void Match_ExcessPrefix()
{
RemoteExecutor.Invoke(() =>
Expand Down