From 9c94e937e859e1198d24d07c0c4f1abf7ed34c20 Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Wed, 22 Apr 2020 05:28:36 +0200 Subject: [PATCH 01/10] Enable dotnet test --- .config/dotnet-tools.json | 6 -- Directory.Build.targets | 1 + .../building/libraries/code-coverage.md | 20 ++--- eng/Versions.props | 1 + eng/referenceFromRuntime.targets | 4 +- eng/references.targets | 2 +- eng/testing/.runsettings | 55 ++++++++++++++ eng/testing/coverage.props | 14 ---- eng/testing/coverage.targets | 73 +++++++++---------- eng/testing/runsettings.targets | 41 +++++++++++ eng/testing/runtimeConfiguration.targets | 2 +- eng/testing/tests.props | 13 ++++ eng/testing/tests.targets | 24 +----- eng/testing/xunit/vstest.props | 15 ---- eng/testing/xunit/vstest.targets | 36 --------- eng/testing/xunit/xunit.console.props | 17 ----- eng/testing/xunit/xunit.props | 26 ++++++- eng/testing/xunit/xunit.targets | 15 +--- eng/versioning.targets | 4 +- global.json | 4 +- .../Common/tests/Common.Tests.csproj | 1 - src/libraries/Directory.Build.props | 24 +++--- src/libraries/Directory.Build.targets | 4 +- ...osoft.XmlSerializer.Generator.Tests.csproj | 1 - .../System.Transactions.Local.Tests.csproj | 1 - src/libraries/tests.proj | 26 ++++--- 26 files changed, 211 insertions(+), 219 deletions(-) create mode 100644 eng/testing/.runsettings delete mode 100644 eng/testing/coverage.props create mode 100644 eng/testing/runsettings.targets delete mode 100644 eng/testing/xunit/vstest.props delete mode 100644 eng/testing/xunit/vstest.targets diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 39540bb21cc957..0626e6dbbb19bb 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -2,12 +2,6 @@ "version": 1, "isRoot": true, "tools": { - "coverlet.console": { - "version": "1.7.0", - "commands": [ - "coverlet" - ] - }, "dotnet-reportgenerator-globaltool": { "version": "4.5.2", "commands": [ diff --git a/Directory.Build.targets b/Directory.Build.targets index 60d17334bd688e..2d72cd0f912847 100644 --- a/Directory.Build.targets +++ b/Directory.Build.targets @@ -26,4 +26,5 @@ preview latest + diff --git a/docs/workflow/building/libraries/code-coverage.md b/docs/workflow/building/libraries/code-coverage.md index a2b288673de2eb..1e504ebb8945a3 100644 --- a/docs/workflow/building/libraries/code-coverage.md +++ b/docs/workflow/building/libraries/code-coverage.md @@ -28,15 +28,15 @@ An issue need not be addressed in its entirety. We happily accept contributions You can perform code coverage runs for the entire repository locally by adding the `coverage` switch (assuming that source and test assemblies are already built): - build -test -coverage + build libs -test -coverage This runs the tests and generates the full code coverage report. The resulting index.htm file providing the results of the run should be available at: artifacts\coverage\index.htm -You can also build and test with code coverage for a particular test project rather than for the whole repo with the ```/p:Coverage=true``` argument: +You can also build and test with code coverage for a particular test project rather than for the whole repo with the `--collect "XPlat code coverage"` switch: - dotnet build /t:Test /p:Coverage=true + dotnet test -f netcoreapp5.0 --collect "XPlat code coverage" The results for this one library will then be available in this index.htm file, where $(OutDir) is the directory where the binaries were generated. @@ -44,24 +44,14 @@ The results for this one library will then be available in this index.htm file, For example, to build, test, and get code coverage results for the System.Diagnostics.Debug library, from the root of the repo one can do: - cd src\System.Diagnostics.Debug\tests\ - dotnet build /t:Test /p:Coverage=true + dotnet test src\libraries\System.Diagnostics.Debug -f netcoreapp5.0 --collect "XPlat code coverage" And then once the run completes: $(OutDir)\report\index.htm -**Note:** If you only want to measure the coverage of your local changes (that haven't been pushed to git), run: - - dotnet build /t:Test /p:Coverage=true /p:CoverageSourceLink=false - - ## Code coverage with System.Private.CoreLib code -Some of the libraries for which contracts and tests live in the corefx repo are actually fully or partially implemented in the core runtime library in another repo, e.g. the implementation that backs the System.Runtime contract is in System.Private.CoreLib.dll in either the coreclr or corert repo. Test projects for code that lives, fully or partially, in System.Private.CoreLib, should have the property `TestRuntime` set to `true` in order to obtain proper code coverage reports. +Some of the libraries for which contracts and tests live in libraries are actually fully or partially implemented in the core runtime library, e.g. the implementation that backs the System.Runtime contract is in System.Private.CoreLib.dll. Test projects for code that lives, fully or partially, in System.Private.CoreLib, should have the property `TestRuntime` set to `true` in order to obtain proper code coverage reports. If the test project does not set the property `TestRuntime` to `true` and you want to collect code coverage that includes types in System.Private.CoreLib.dll add `/p:TestRuntime=true` to the coverage build command listed above. - -If you want to get coverage report against a private build of System.Private.CoreLib (// TODO //). - -The build and test projects take care of copying assemblies and PDBs as needed for coverage runs. The resulting code coverage report should now also include details for System.Private.CoreLib. diff --git a/eng/Versions.props b/eng/Versions.props index 230672aec71957..5406010c6181d4 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -116,6 +116,7 @@ 16.7.0-preview-20200429-01 1.0.0-prerelease.20230.6 2.4.1 + 1.2.0 2.0.5 12.0.3 4.12.0 diff --git a/eng/referenceFromRuntime.targets b/eng/referenceFromRuntime.targets index 873e7e5ff40192..26e107a13101c2 100644 --- a/eng/referenceFromRuntime.targets +++ b/eng/referenceFromRuntime.targets @@ -19,7 +19,7 @@ + Condition="'$(IsTestProject)' != 'true' and '$(IsTestSupportProject)' != 'true' and '@(ReferenceFromRuntime)' != ''"> @@ -65,7 +65,7 @@ <_referencePathFromRuntime Include="@(RuntimeFiles)" Private="false" /> <_referencePathFromRuntime Include="@(_referencePathFromRestoredRuntime)" Private="false" /> - <_referencePathFromRuntime Include="@(ReferenceFromRuntime->'$(RuntimePath)%(Identity).dll')" Condition="'$(IsTestProject)' == 'true'" /> + <_referencePathFromRuntime Include="@(ReferenceFromRuntime->'$(RuntimePath)%(Identity).dll')" Condition="'$(IsTestProject)' == 'true' or '$(IsTestSupportProject)' == 'true'" /> <_referencePathFromRuntimeByFileName Include="@(_referencePathFromRuntime->'%(FileName)')" Condition="'%(_referencePathFromRuntime.Extension)' == '.dll'" > %(Identity) diff --git a/eng/references.targets b/eng/references.targets index 0c104db2bb27bc..405fd07799f594 100644 --- a/eng/references.targets +++ b/eng/references.targets @@ -36,7 +36,7 @@ - + diff --git a/eng/testing/.runsettings b/eng/testing/.runsettings new file mode 100644 index 00000000000000..a2af8f6f18bdea --- /dev/null +++ b/eng/testing/.runsettings @@ -0,0 +1,55 @@ + + + + + 300000 + + .\ + + .\ + + $$MAXCPUCOUNT$$ + + $$TARGETPLATFORM$$ + + $$DISABLEPARALLELIZATION$$ + + $$DISABLEAPPDOMAIN$$ + + $$TESTCASEFILTER$$ + $$DOTNETHOSTPATH$$ + + + $$DEVPATH$$ + + + + + + + testResults.trx + + + + + testResults.html + + + + + + + + + $$COVERAGE_INCLUDE$$ + $$COVERAGE_EXCLUDEBYFILE$$ + $$COVERAGE_INCLUDEDIRECTORY$$ + opencover + false + true + false + + + + + \ No newline at end of file diff --git a/eng/testing/coverage.props b/eng/testing/coverage.props deleted file mode 100644 index 7af30267a5625a..00000000000000 --- a/eng/testing/coverage.props +++ /dev/null @@ -1,14 +0,0 @@ - - - coverage.xml - 0 - line,branch,method - opencover - true - normal - - $(CoverageOutputPath) - Html - Info - - diff --git a/eng/testing/coverage.targets b/eng/testing/coverage.targets index 3638e3efd7ca6b..b369d72add3854 100644 --- a/eng/testing/coverage.targets +++ b/eng/testing/coverage.targets @@ -1,39 +1,6 @@ - - - "$(TargetFileName)" --target "$(RunCommand)" --targetargs "$(RunArguments)" --format "$(CoverageFormat)" --output "$(CoverageOutputPath)" --threshold "$(CoverageThreshold)" --threshold-type "$(CoverageThresholdType)" --verbosity "$(CoverageVerbosity)" - $(RunArguments) --use-source-link - "$(DotNetTool)" tool run coverlet - - - - - $([MSBuild]::NormalizeDirectory('$(OutDir)', 'report')) - $([MSBuild]::NormalizePath('$(CoverageReportDir)', 'index.htm')) - "$(DotNetTool)" tool run reportgenerator "-reports:$(CoverageReportInputPath)" "-targetdir:$(CoverageReportDir.TrimEnd('\/'))" "-reporttypes:$(CoverageReportTypes)" "-verbosity:$(CoverageReportVerbosity)" - - - - - - - - --exclude-by-file @(CoverageExcludeFile -> '"%(Identity)"', ' --exclude-by-file ') - $(RunArguments) $(CoverageExcludeByFileFilter) - - - - --include-directory @(CoverageProbePath -> '"$(RunScriptHostDir)%(Identity)"', ' --include-directory ') - $(RunArguments) $(IncludeDirectoriesFilter) - - - - --exclude @(CoverageExclude -> '"%(Identity)"', ' --exclude ') - $(RunArguments) $(CoverageExcludeFilter) - - - <_CoverageAssemblies Include="$(AssemblyBeingTested)" /> - <_CoverageAssemblies Include="System.Private.CoreLib" Condition="'$(TestRuntime)' == 'true'" /> - <_CoverageAssemblies Include="@(AssembliesBeingTested)" /> - <_CoverageAssemblies Include="$(CoverageAssemblies)" Condition="'$(CoverageAssemblies)' != ''" /> + + + + - - --include @(_CoverageAssemblies -> '"[%(Identity)]*"', ' --include ') - $(RunArguments) $(CoverageFilter) + + @(CoverageInclude -> '[%(Identity)]*', ',') + + + + @(CoverageExcludeByFile -> '%(Identity)', ',') + + + @(CoverageIncludeDirectory -> '$(TestHostRootPath)%(Identity)', ',') + + + + + + + + + + %(CoverageOutputFile.Identity) + Html + Info + $([MSBuild]::NormalizeDirectory('$(OutDir)', 'report')) + "$(DotNetTool)" tool run reportgenerator "-reports:$(CoverageReportInputPath)" "-targetdir:$(CoverageReportDir.TrimEnd('\/'))" "-reporttypes:$(CoverageReportTypes)" "-verbosity:$(CoverageReportVerbosity)" + + + + diff --git a/eng/testing/runsettings.targets b/eng/testing/runsettings.targets new file mode 100644 index 00000000000000..76171c642228fc --- /dev/null +++ b/eng/testing/runsettings.targets @@ -0,0 +1,41 @@ + + + $(MSBuildThisFileDirectory).runsettings + $(OutDir).runsettings + GenerateRunSettingsFile;$(PrepareForRunDependsOn) + + + + + <_testFilter Condition="'$(_withCategories)' != ''">$(_withCategories.Replace(';', '&amp;category=')) + <_testFilter Condition="'$(_withoutCategories)' != ''">$(_testFilter)$(_withoutCategories.Replace(';', '&amp;category!=')) + <_testFilter>$(_testFilter.Trim('&amp;')) + + + + + $([System.IO.File]::ReadAllText('$(RunSettingsInputFilePath)')) + $(RunSettingsFileContent.Replace('$$COVERAGE_INCLUDE$$', '$(CoverageIncludeFilter)')) + $(RunSettingsFileContent.Replace('$$COVERAGE_EXCLUDEBYFILE$$', '$(CoverageExcludeByFileFilter)')) + $(RunSettingsFileContent.Replace('$$COVERAGE_INCLUDEDIRECTORY$$', '$(CoverageIncludeDirectoryFilter)')) + $(RunSettingsFileContent.Replace('$$MAXCPUCOUNT$$', '1')) + $(RunSettingsFileContent.Replace('$$MAXCPUCOUNT$$', '0')) + $(RunSettingsFileContent.Replace('$$TARGETPLATFORM$$', '$(TargetArchitecture)')) + $(RunSettingsFileContent.Replace('$$DISABLEPARALLELIZATION$$', '$([MSBuild]::ValueOrDefault('$(TestDisableParallelization)', 'false'))')) + $(RunSettingsFileContent.Replace('$$DISABLEAPPDOMAIN$$', '$([MSBuild]::ValueOrDefault('$(TestDisableAppDomain)', 'false'))')) + $(RunSettingsFileContent.Replace('$$TESTCASEFILTER$$', '$(_testFilter)')) + $(RunSettingsFileContent.Replace('$$DEVPATH$$', '$(TestHostRootPath)')) + $(RunSettingsFileContent.Replace('$$DOTNETHOSTPATH$$', '$(TestHostRootPath)$([System.IO.Path]::GetFileName('$(DotNetTool)'))')) + + + + + + + $(RunSettingsOutputFilePath) + + + diff --git a/eng/testing/runtimeConfiguration.targets b/eng/testing/runtimeConfiguration.targets index fdb81abbb18929..d22759523fdbff 100644 --- a/eng/testing/runtimeConfiguration.targets +++ b/eng/testing/runtimeConfiguration.targets @@ -4,7 +4,7 @@ $(MSBuildThisFileDirectory)netfx.exe.config - true + true diff --git a/eng/testing/tests.props b/eng/testing/tests.props index dfaabfa18db038..0c792a6d116db5 100644 --- a/eng/testing/tests.props +++ b/eng/testing/tests.props @@ -5,6 +5,19 @@ Build $(TestDependsOn);GenerateRunScript;RunTests + true + + + + + <_withCategories Condition="'$(WithCategories)' != ''">;$(WithCategories.Trim(';')) + <_withoutCategories Condition="'$(WithoutCategories)' != ''">;$(WithoutCategories.Trim(';')) + + all + <_withCategories Condition="'$(TestScope)' == 'outerloop'">$(_withCategories);OuterLoop + <_withoutCategories Condition="'$(ArchiveTests)' == 'true'">$(_withoutCategories);IgnoreForCI + <_withoutCategories Condition="'$(TestScope)' == '' or '$(TestScope)' == 'innerloop'">$(_withoutCategories);OuterLoop + <_withoutCategories Condition="!$(_withCategories.Contains('failing'))">$(_withoutCategories);failing diff --git a/eng/testing/tests.targets b/eng/testing/tests.targets index b90176bd9bba2a..dbc75c63553afe 100644 --- a/eng/testing/tests.targets +++ b/eng/testing/tests.targets @@ -18,7 +18,6 @@ $(RunScriptHostDir)dotnet.exe $(RunScriptHostDir)dotnet - @@ -74,28 +73,10 @@ TemplatePath="$(RunScriptInputPath)" OutputPath="$(RunScriptOutputPath)" /> - - - - - - - - - - - true - - - - - + - + "$(RunScriptOutputPath)" --runtime-path "$(TestHostRootPath.TrimEnd('\/'))" $(RunTestsCommand) --rsp-file "$(TestRspFile)" @@ -231,6 +212,7 @@ + diff --git a/eng/testing/xunit/vstest.props b/eng/testing/xunit/vstest.props deleted file mode 100644 index 152ca6e67db741..00000000000000 --- a/eng/testing/xunit/vstest.props +++ /dev/null @@ -1,15 +0,0 @@ - - - - en - testResults.trx - testResults.html - $(HtmlTestResultsName) - - - - - - - - diff --git a/eng/testing/xunit/vstest.targets b/eng/testing/xunit/vstest.targets deleted file mode 100644 index 3a638a442c0998..00000000000000 --- a/eng/testing/xunit/vstest.targets +++ /dev/null @@ -1,36 +0,0 @@ - - - "$(DotNetTool)" - dotnet - test $(TargetFileName) - - - $(RunArguments) --nologo - $(RunArguments) --logger "trx;LogFileName=$(TrxTestResultsName)" - $(RunArguments) --logger "html;LogFileName=$(HtmlTestResultsName)" - $(RunArguments) --framework $(TargetFramework) - $(RunArguments) --platform $(TargetArchitecture) - $(RunArguments) --parallel - $(RunArguments) --blame - - - <_testFilter Condition="'$(_withCategories)' != ''">$(_withCategories.Replace(';', '&category=')) - <_testFilter Condition="'$(_withoutCategories)' != ''">$(_testFilter)$(_withoutCategories.Replace(';', '&category!=')) - - <_testFilter Condition="'$(TargetOS)' == 'Windows_NT'">$(_testFilter.Replace('!=', '^!=')) - <_testFilter>$(_testFilter.Trim('&')) - - <_testFilter Condition="'$(TestFilter)' != ''">$(_testFilter)&$(TestFilter.Replace('!=', '^!=')) - - $(RunArguments) --filter "($(_testFilter))" - - - $(RunArguments) $(XUnitOptions) - - - $(RunSettingsOptions) RunConfiguration.DisableParallelization=true - $(RunSettingsOptions) RunConfiguration.DisableAppDomain=true - $(RunSettingsOptions) RunConfiguration.DotNetHostPath="$(RunScriptHost)" - $(RunArguments) --$(RunSettingsOptions) - - \ No newline at end of file diff --git a/eng/testing/xunit/xunit.console.props b/eng/testing/xunit/xunit.console.props index 59554280e9886e..36950a75c5bfe2 100644 --- a/eng/testing/xunit/xunit.console.props +++ b/eng/testing/xunit/xunit.console.props @@ -2,27 +2,10 @@ true testResults.xml - $(MSBuildThisFileDirectory)xunit.runner.json - - - - - - - - - - diff --git a/eng/testing/xunit/xunit.props b/eng/testing/xunit/xunit.props index 0b6289b9a22669..4af05170f3acd5 100644 --- a/eng/testing/xunit/xunit.props +++ b/eng/testing/xunit/xunit.props @@ -1,6 +1,8 @@ - xunit.console + + en + $(MSBuildThisFileDirectory)xunit.runner.json @@ -11,6 +13,24 @@ - - + + + + + + + + + + + + + + diff --git a/eng/testing/xunit/xunit.targets b/eng/testing/xunit/xunit.targets index 348be251a9fb16..2341f354c9c0fb 100644 --- a/eng/testing/xunit/xunit.targets +++ b/eng/testing/xunit/xunit.targets @@ -1,16 +1,3 @@ - - - <_withCategories Condition="'$(WithCategories)' != ''">;$(WithCategories.Trim(';')) - <_withoutCategories Condition="'$(WithoutCategories)' != ''">;$(WithoutCategories.Trim(';')) - - all - <_withCategories Condition="'$(TestScope)' == 'outerloop'">$(_withCategories);OuterLoop - <_withoutCategories Condition="'$(ArchiveTests)' == 'true'">$(_withoutCategories);IgnoreForCI - <_withoutCategories Condition="'$(TestScope)' == '' or '$(TestScope)' == 'innerloop'">$(_withoutCategories);OuterLoop - <_withoutCategories Condition="!$(_withCategories.Contains('failing'))">$(_withoutCategories);failing - - - - + diff --git a/eng/versioning.targets b/eng/versioning.targets index 18dc8a1163d699..21a442a03f3c0d 100644 --- a/eng/versioning.targets +++ b/eng/versioning.targets @@ -10,7 +10,7 @@ - + @@ -23,7 +23,7 @@ + Condition="'$(IsDotNetFrameworkProductAssembly)' == 'true' and '$(IsTestProject)' != 'true' and '$(IsTestSupportProject)' != 'true'"> diff --git a/global.json b/global.json index 5ee2d38f07ea54..ee9fee7285da43 100644 --- a/global.json +++ b/global.json @@ -1,11 +1,11 @@ { "sdk": { - "version": "5.0.100-preview.4.20202.8", + "version": "5.0.100-preview.4.20217.10", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "5.0.100-preview.4.20202.8" + "dotnet": "5.0.100-preview.4.20217.10" }, "native-tools": { "cmake": "3.14.2", diff --git a/src/libraries/Common/tests/Common.Tests.csproj b/src/libraries/Common/tests/Common.Tests.csproj index 82c731c02af231..313fb9f76de7e4 100644 --- a/src/libraries/Common/tests/Common.Tests.csproj +++ b/src/libraries/Common/tests/Common.Tests.csproj @@ -1,7 +1,6 @@ true - false true $(NetCoreAppCurrent)-Windows_NT;$(NetCoreAppCurrent)-Linux;$(NetCoreAppCurrent)-OSX annotations diff --git a/src/libraries/Directory.Build.props b/src/libraries/Directory.Build.props index 4540c81857f3c3..573fb92e5966d1 100644 --- a/src/libraries/Directory.Build.props +++ b/src/libraries/Directory.Build.props @@ -55,10 +55,9 @@ false true - true - false + false @@ -314,35 +313,34 @@ $(TestHostRuntimePath)PlatformManifest.txt - + - + - + true - true + true true - - - + + - + - + false @@ -354,12 +352,12 @@ feature unless they explicitly opt in to it. (Source projects explicitly reference their dependencies, so they can't inadvertently take a dependency on it.) --> - + - false + false true diff --git a/src/libraries/Directory.Build.targets b/src/libraries/Directory.Build.targets index 63c66e0226b335..803668948a1608 100644 --- a/src/libraries/Directory.Build.targets +++ b/src/libraries/Directory.Build.targets @@ -31,7 +31,7 @@ Tests which wish to control this should set TestStrongNameKeyId. --> MicrosoftAspNetCore Open - $(TestStrongNameKeyId) + $(TestStrongNameKeyId) @@ -51,7 +51,7 @@ - true + true true diff --git a/src/libraries/Microsoft.XmlSerializer.Generator/tests/Microsoft.XmlSerializer.Generator.Tests.csproj b/src/libraries/Microsoft.XmlSerializer.Generator/tests/Microsoft.XmlSerializer.Generator.Tests.csproj index 88cd7f626733ed..7cffe3caa31032 100644 --- a/src/libraries/Microsoft.XmlSerializer.Generator/tests/Microsoft.XmlSerializer.Generator.Tests.csproj +++ b/src/libraries/Microsoft.XmlSerializer.Generator/tests/Microsoft.XmlSerializer.Generator.Tests.csproj @@ -2,7 +2,6 @@ $(DefineConstants);XMLSERIALIZERGENERATORTESTS $(NetCoreAppCurrent) - false true diff --git a/src/libraries/System.Transactions.Local/tests/System.Transactions.Local.Tests.csproj b/src/libraries/System.Transactions.Local/tests/System.Transactions.Local.Tests.csproj index 750635dac0a537..97efb68bab6ad5 100644 --- a/src/libraries/System.Transactions.Local/tests/System.Transactions.Local.Tests.csproj +++ b/src/libraries/System.Transactions.Local/tests/System.Transactions.Local.Tests.csproj @@ -1,6 +1,5 @@ - false true $(NetCoreAppCurrent) diff --git a/src/libraries/tests.proj b/src/libraries/tests.proj index f2e7cb3a071528..a22aaf27819b2d 100644 --- a/src/libraries/tests.proj +++ b/src/libraries/tests.proj @@ -1,15 +1,22 @@ - - - true + false ErrorAndContinue BuildAllProjects=true - $(ArtifactsBinDir)*.Tests/**/coverage.xml + $(TraversalGlobalProperties);VSTestCollect="XPlat code coverage" + $(ArtifactsBinDir)*.Tests\*\coverage.opencover.xml $(ArtifactsDir)coverage + true + + + + + + + @@ -20,14 +27,9 @@ Condition="'$(BuildAllConfigurations)' == 'true'" /> - - - - - + DependsOnTargets="GenerateCoverageReport" /> From a8b8be389dfffa52d6784ff0c5a0c1506b749fb4 Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Wed, 22 Apr 2020 20:34:10 +0200 Subject: [PATCH 02/10] Update docs --- docs/workflow/building/libraries/code-coverage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/workflow/building/libraries/code-coverage.md b/docs/workflow/building/libraries/code-coverage.md index 1e504ebb8945a3..0cc5c452d123dd 100644 --- a/docs/workflow/building/libraries/code-coverage.md +++ b/docs/workflow/building/libraries/code-coverage.md @@ -28,7 +28,7 @@ An issue need not be addressed in its entirety. We happily accept contributions You can perform code coverage runs for the entire repository locally by adding the `coverage` switch (assuming that source and test assemblies are already built): - build libs -test -coverage + build libs.tests -test -coverage This runs the tests and generates the full code coverage report. The resulting index.htm file providing the results of the run should be available at: From 1fb78fd962f6890f4da6a8b59aea741434cb8aa1 Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Thu, 23 Apr 2020 19:55:18 +0200 Subject: [PATCH 03/10] Use vstest in F5 scenarios --- eng/testing/.runsettings | 5 +++ eng/testing/coverage.targets | 2 +- eng/testing/launchSettings.json | 19 ----------- eng/testing/launchSettings.targets | 22 ------------- eng/testing/tests.targets | 14 +++------ eng/testing/xunit/xunit.console.targets | 42 ++++++++++--------------- eng/testing/xunit/xunit.props | 2 +- eng/testing/xunit/xunit.targets | 15 +++++++++ src/libraries/Directory.Build.props | 1 - src/libraries/Directory.Build.targets | 1 - 10 files changed, 44 insertions(+), 79 deletions(-) delete mode 100644 eng/testing/launchSettings.json delete mode 100644 eng/testing/launchSettings.targets diff --git a/eng/testing/.runsettings b/eng/testing/.runsettings index a2af8f6f18bdea..86b891ed3ec0c5 100644 --- a/eng/testing/.runsettings +++ b/eng/testing/.runsettings @@ -35,6 +35,11 @@ testResults.html + + + Minimal + + diff --git a/eng/testing/coverage.targets b/eng/testing/coverage.targets index b369d72add3854..8f04995f3e6010 100644 --- a/eng/testing/coverage.targets +++ b/eng/testing/coverage.targets @@ -38,7 +38,7 @@ Condition="'$(VSTestCollect)' == 'XPlat code coverage' and '$(SkipCoverageReport)' != 'true'" AfterTargets="VSTest"> - + diff --git a/eng/testing/launchSettings.json b/eng/testing/launchSettings.json deleted file mode 100644 index 62d099feb3b7d8..00000000000000 --- a/eng/testing/launchSettings.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "profiles": { - ".NET Core xUnit Console": { - "commandName": "Executable", - "executablePath": "$(TestHostRootPath)dotnet.exe", - "commandLineArgs": "$(RunArguments) -parallel none", - "workingDirectory": "$(RunWorkingDirectory)" - }, - ".NET Framework xUnit Console": { - "commandName": "Executable", - "executablePath": "$(RunWorkingDirectory)$(RunCommand)", - "commandLineArgs": "$(RunArguments) -parallel none", - "workingDirectory": "$(RunWorkingDirectory)", - "environmentVariables": { - "DEVPATH": "$(TestHostRootPath)" - } - } - } -} diff --git a/eng/testing/launchSettings.targets b/eng/testing/launchSettings.targets deleted file mode 100644 index 6caf9475304570..00000000000000 --- a/eng/testing/launchSettings.targets +++ /dev/null @@ -1,22 +0,0 @@ - - - - $(MSBuildThisFileDirectory)launchSettings.json - $([MSBuild]::NormalizePath('$(MSBuildProjectDirectory)', '$(AppDesignerFolder)', 'launchSettings.json')) - GenerateLaunchSettingsFile;$(PrepareForRunDependsOn); - - - - - - - \ No newline at end of file diff --git a/eng/testing/tests.targets b/eng/testing/tests.targets index dbc75c63553afe..faaffc411b34b3 100644 --- a/eng/testing/tests.targets +++ b/eng/testing/tests.targets @@ -1,7 +1,5 @@ - $(OutDir) - RunnerTemplate.cmd RunnerTemplate.sh AppleRunnerTemplate.sh @@ -37,13 +35,11 @@ - $(RunArguments) %RSP_FILE% - $(RunArguments) $RSP_FILE - - - $([MSBuild]::Escape('$(RunArguments)')) + $(RunScriptCommand) %RSP_FILE% + $(RunScriptCommand) $RSP_FILE - $(RunCommand) $(RunArguments) + + $([MSBuild]::Escape('$(RunScriptCommand)')) @@ -92,7 +88,7 @@ - $(RunWorkingDirectory)$(TestResultsName) + $(OutputPath)$(TestResultsName) One or more tests failed while running tests from '$(TestProjectName)'. $(TestRunErrorMessage) Please check $(TestResultsPath) for details! diff --git a/eng/testing/xunit/xunit.console.targets b/eng/testing/xunit/xunit.console.targets index 059eede7dcf42b..1b7cd52d56015e 100644 --- a/eng/testing/xunit/xunit.console.targets +++ b/eng/testing/xunit/xunit.console.targets @@ -1,23 +1,27 @@ - $(TargetFileName) - $(RunArguments) -xml $(TestResultsName) - $(RunArguments) -nologo - $(RunArguments) -nocolor + <_depsFileArgument Condition="'$(GenerateDependencyFile)' == 'true'">--depsfile $(AssemblyName).deps.json + "$(RunScriptHost)" exec --runtimeconfig $(AssemblyName).runtimeconfig.json $(_depsFileRunArgument) xunit.console.dll + xunit.console.exe + + $(RunScriptCommand) $(TargetFileName) + $(RunScriptCommand) -xml $(TestResultsName) + $(RunScriptCommand) -nologo + $(RunScriptCommand) -nocolor - $(RunArguments) -maxthreads 1 - $(RunArguments) -method $(XUnitMethodName) - $(RunArguments) -class $(XUnitClassName) - $(RunArguments) -verbose - $(RunArguments) -noappdomain + $(RunScriptCommand) -maxthreads 1 + $(RunScriptCommand) -method $(XUnitMethodName) + $(RunScriptCommand) -class $(XUnitClassName) + $(RunScriptCommand) -verbose + $(RunScriptCommand) -noappdomain - $(RunArguments)$(_withCategories.Replace(';', ' -trait category=')) - $(RunArguments)$(_withoutCategories.Replace(';', ' -notrait category=')) + $(RunScriptCommand)$(_withCategories.Replace(';', ' -trait category=')) + $(RunScriptCommand)$(_withoutCategories.Replace(';', ' -notrait category=')) - $(RunArguments) $(XUnitOptions) + $(RunScriptCommand) $(XUnitOptions) @@ -27,7 +31,7 @@ <_testRunnerConfigSourceFile Include="$(TargetDir)$(TargetName).exe.config" /> - <_testRunnerConfigDestFile Include="$(TargetDir)$(_testRunnerName).config" /> + <_testRunnerConfigDestFile Include="$(TargetDir)xunit.console.exe.config" /> - - - "$(RunScriptHost)" - <_depsFileRunArgument Condition="'$(GenerateDependencyFile)' == 'true'">--depsfile $(AssemblyName).deps.json - exec --runtimeconfig $(AssemblyName).runtimeconfig.json $(_depsFileRunArgument) xunit.console.dll $(RunArguments) - - - - <_testRunnerName>xunit.console.exe - $(_testRunnerName) - - diff --git a/eng/testing/xunit/xunit.props b/eng/testing/xunit/xunit.props index 4af05170f3acd5..4fed5c3990f290 100644 --- a/eng/testing/xunit/xunit.props +++ b/eng/testing/xunit/xunit.props @@ -19,7 +19,7 @@ diff --git a/eng/testing/xunit/xunit.targets b/eng/testing/xunit/xunit.targets index 2341f354c9c0fb..4d890c53b928e6 100644 --- a/eng/testing/xunit/xunit.targets +++ b/eng/testing/xunit/xunit.targets @@ -1,3 +1,18 @@ + + + $(OutDir) + + + + "$(DotNetTool)" + test $(TargetPath) --settings $(OutDir).runsettings + + + + "$(DevEnvDir)Extensions\TestPlatform\vstest.console.exe" + $(TargetPath) --settings:$(OutDir).runsettings + + diff --git a/src/libraries/Directory.Build.props b/src/libraries/Directory.Build.props index 573fb92e5966d1..891b819f79b8c6 100644 --- a/src/libraries/Directory.Build.props +++ b/src/libraries/Directory.Build.props @@ -324,7 +324,6 @@ true true - true diff --git a/src/libraries/Directory.Build.targets b/src/libraries/Directory.Build.targets index 803668948a1608..435e3f5be249fa 100644 --- a/src/libraries/Directory.Build.targets +++ b/src/libraries/Directory.Build.targets @@ -142,7 +142,6 @@ - From 44ad83db37f9895228d34fde8dc5bf41fd1b8fbb Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Thu, 23 Apr 2020 22:25:28 +0200 Subject: [PATCH 04/10] Make dotnet test without framework switch work --- eng/testing/runsettings.targets | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/testing/runsettings.targets b/eng/testing/runsettings.targets index 76171c642228fc..319046fb744b3d 100644 --- a/eng/testing/runsettings.targets +++ b/eng/testing/runsettings.targets @@ -2,11 +2,12 @@ $(MSBuildThisFileDirectory).runsettings $(OutDir).runsettings + + $(RunSettingsOutputFilePath) GenerateRunSettingsFile;$(PrepareForRunDependsOn) - <_testFilter Condition="'$(_withCategories)' != ''">$(_withCategories.Replace(';', '&amp;category=')) <_testFilter Condition="'$(_withoutCategories)' != ''">$(_testFilter)$(_withoutCategories.Replace(';', '&amp;category!=')) <_testFilter>$(_testFilter.Trim('&amp;')) From 2c5d6fd2645ad319260bca06beb68b9ec6c34102 Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Thu, 23 Apr 2020 22:31:48 +0200 Subject: [PATCH 05/10] Code cleanup --- eng/testing/runsettings.targets | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/eng/testing/runsettings.targets b/eng/testing/runsettings.targets index 319046fb744b3d..7f5a726c7aaf26 100644 --- a/eng/testing/runsettings.targets +++ b/eng/testing/runsettings.targets @@ -16,17 +16,17 @@ $([System.IO.File]::ReadAllText('$(RunSettingsInputFilePath)')) - $(RunSettingsFileContent.Replace('$$COVERAGE_INCLUDE$$', '$(CoverageIncludeFilter)')) - $(RunSettingsFileContent.Replace('$$COVERAGE_EXCLUDEBYFILE$$', '$(CoverageExcludeByFileFilter)')) - $(RunSettingsFileContent.Replace('$$COVERAGE_INCLUDEDIRECTORY$$', '$(CoverageIncludeDirectoryFilter)')) $(RunSettingsFileContent.Replace('$$MAXCPUCOUNT$$', '1')) $(RunSettingsFileContent.Replace('$$MAXCPUCOUNT$$', '0')) - $(RunSettingsFileContent.Replace('$$TARGETPLATFORM$$', '$(TargetArchitecture)')) - $(RunSettingsFileContent.Replace('$$DISABLEPARALLELIZATION$$', '$([MSBuild]::ValueOrDefault('$(TestDisableParallelization)', 'false'))')) - $(RunSettingsFileContent.Replace('$$DISABLEAPPDOMAIN$$', '$([MSBuild]::ValueOrDefault('$(TestDisableAppDomain)', 'false'))')) - $(RunSettingsFileContent.Replace('$$TESTCASEFILTER$$', '$(_testFilter)')) - $(RunSettingsFileContent.Replace('$$DEVPATH$$', '$(TestHostRootPath)')) - $(RunSettingsFileContent.Replace('$$DOTNETHOSTPATH$$', '$(TestHostRootPath)$([System.IO.Path]::GetFileName('$(DotNetTool)'))')) + $(RunSettingsFileContent.Replace('$$COVERAGE_INCLUDE$$', '$(CoverageIncludeFilter)') + .Replace('$$COVERAGE_EXCLUDEBYFILE$$', '$(CoverageExcludeByFileFilter)') + .Replace('$$COVERAGE_INCLUDEDIRECTORY$$', '$(CoverageIncludeDirectoryFilter)') + .Replace('$$TARGETPLATFORM$$', '$(TargetArchitecture)') + .Replace('$$DISABLEPARALLELIZATION$$', '$([MSBuild]::ValueOrDefault('$(TestDisableParallelization)', 'false'))') + .Replace('$$DISABLEAPPDOMAIN$$', '$([MSBuild]::ValueOrDefault('$(TestDisableAppDomain)', 'false'))') + .Replace('$$TESTCASEFILTER$$', '$(_testFilter)') + .Replace('$$DEVPATH$$', '$(TestHostRootPath)') + .Replace('$$DOTNETHOSTPATH$$', '$(TestHostRootPath)$([System.IO.Path]::GetFileName('$(DotNetTool)'))')) Date: Thu, 23 Apr 2020 23:11:26 +0200 Subject: [PATCH 06/10] Only run code coverage conditionally --- docs/workflow/building/libraries/code-coverage.md | 6 +++--- eng/testing/.runsettings | 2 +- eng/testing/coverage.targets | 6 ++---- eng/testing/runsettings.targets | 4 +++- src/libraries/tests.proj | 5 ++--- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/docs/workflow/building/libraries/code-coverage.md b/docs/workflow/building/libraries/code-coverage.md index 0cc5c452d123dd..ad67ed4301d3e4 100644 --- a/docs/workflow/building/libraries/code-coverage.md +++ b/docs/workflow/building/libraries/code-coverage.md @@ -34,9 +34,9 @@ This runs the tests and generates the full code coverage report. The resulting i artifacts\coverage\index.htm -You can also build and test with code coverage for a particular test project rather than for the whole repo with the `--collect "XPlat code coverage"` switch: +You can also build and test with code coverage for a particular test project rather than for the whole repo with the `/p:Coverage=true` property: - dotnet test -f netcoreapp5.0 --collect "XPlat code coverage" + dotnet test -f netcoreapp5.0 /p:Coverage=true The results for this one library will then be available in this index.htm file, where $(OutDir) is the directory where the binaries were generated. @@ -44,7 +44,7 @@ The results for this one library will then be available in this index.htm file, For example, to build, test, and get code coverage results for the System.Diagnostics.Debug library, from the root of the repo one can do: - dotnet test src\libraries\System.Diagnostics.Debug -f netcoreapp5.0 --collect "XPlat code coverage" + dotnet test src\libraries\System.Diagnostics.Debug -f netcoreapp5.0 /p:Coverage=true And then once the run completes: diff --git a/eng/testing/.runsettings b/eng/testing/.runsettings index 86b891ed3ec0c5..940a874952c3c3 100644 --- a/eng/testing/.runsettings +++ b/eng/testing/.runsettings @@ -44,7 +44,7 @@ - + $$COVERAGE_INCLUDE$$ $$COVERAGE_EXCLUDEBYFILE$$ diff --git a/eng/testing/coverage.targets b/eng/testing/coverage.targets index 8f04995f3e6010..35b89936046574 100644 --- a/eng/testing/coverage.targets +++ b/eng/testing/coverage.targets @@ -1,5 +1,4 @@ - - + ErrorAndContinue BuildAllProjects=true - $(TraversalGlobalProperties);VSTestCollect="XPlat code coverage" $(ArtifactsBinDir)*.Tests\*\coverage.opencover.xml $(ArtifactsDir)coverage true @@ -28,7 +27,7 @@ From f602d6eea54e684dd0523eff9cf89f095f1bf94f Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Fri, 24 Apr 2020 09:18:13 +0200 Subject: [PATCH 07/10] Fix F5 condition for netcoreapp --- eng/testing/xunit/xunit.targets | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/testing/xunit/xunit.targets b/eng/testing/xunit/xunit.targets index 4d890c53b928e6..b14d3b7cbf227c 100644 --- a/eng/testing/xunit/xunit.targets +++ b/eng/testing/xunit/xunit.targets @@ -4,13 +4,13 @@ $(OutDir) - - "$(DotNetTool)" + + $(DotNetTool) test $(TargetPath) --settings $(OutDir).runsettings - "$(DevEnvDir)Extensions\TestPlatform\vstest.console.exe" + $(DevEnvDir)Extensions\TestPlatform\vstest.console.exe $(TargetPath) --settings:$(OutDir).runsettings From df9c1ccef65b7cfb82079a6ee1811a65999e264e Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Fri, 24 Apr 2020 11:43:37 +0200 Subject: [PATCH 08/10] Downgrade sdk to check for helix submission failures --- global.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/global.json b/global.json index ee9fee7285da43..5ee2d38f07ea54 100644 --- a/global.json +++ b/global.json @@ -1,11 +1,11 @@ { "sdk": { - "version": "5.0.100-preview.4.20217.10", + "version": "5.0.100-preview.4.20202.8", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "5.0.100-preview.4.20217.10" + "dotnet": "5.0.100-preview.4.20202.8" }, "native-tools": { "cmake": "3.14.2", From 76963f489a911af02b08b4ffdfabd59d7b02cd81 Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Fri, 24 Apr 2020 13:25:45 +0200 Subject: [PATCH 09/10] Fix wrong conditions --- eng/testing/runtimeConfiguration.targets | 2 +- eng/testing/xunit/xunit.console.targets | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/testing/runtimeConfiguration.targets b/eng/testing/runtimeConfiguration.targets index d22759523fdbff..c92c6ff627dfc0 100644 --- a/eng/testing/runtimeConfiguration.targets +++ b/eng/testing/runtimeConfiguration.targets @@ -4,7 +4,7 @@ $(MSBuildThisFileDirectory)netfx.exe.config - true + true diff --git a/eng/testing/xunit/xunit.console.targets b/eng/testing/xunit/xunit.console.targets index 1b7cd52d56015e..ce234b69e8398f 100644 --- a/eng/testing/xunit/xunit.console.targets +++ b/eng/testing/xunit/xunit.console.targets @@ -1,7 +1,7 @@ <_depsFileArgument Condition="'$(GenerateDependencyFile)' == 'true'">--depsfile $(AssemblyName).deps.json - "$(RunScriptHost)" exec --runtimeconfig $(AssemblyName).runtimeconfig.json $(_depsFileRunArgument) xunit.console.dll + "$(RunScriptHost)" exec --runtimeconfig $(AssemblyName).runtimeconfig.json $(_depsFileArgument) xunit.console.dll xunit.console.exe $(RunScriptCommand) $(TargetFileName) From b2dbc589bdb84b9f322682e3d008b6dfa208e4ee Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Sun, 3 May 2020 21:34:11 +0200 Subject: [PATCH 10/10] Add blame data collector --- eng/testing/.runsettings | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eng/testing/.runsettings b/eng/testing/.runsettings index 940a874952c3c3..138ee92187e6fb 100644 --- a/eng/testing/.runsettings +++ b/eng/testing/.runsettings @@ -55,6 +55,11 @@ false + + + + + \ No newline at end of file