Run PAL tests in CI

docs/workflow/testing/coreclr/unix-test-instructions.md

@@ -81,15 +81,23 @@ PAL tests
 
 Build CoreCLR with PAL tests on the Unix machine:
 
-```sh
-./src/coreclr/build-runtime.sh -skipgenerateversion -nopgooptimize \
-    -cmakeargs -DCLR_CMAKE_BUILD_TESTS=1
+```sh ./build.sh clr.paltests
 ```
 
 Run tests:
 
+To run all tests including disabled tests
+```sh
+./src/coreclr/src/pal/tests/palsuite/runpaltests.sh $(pwd)/artifacts/bin/coreclr/$(uname).x64.Debug/paltests
+# on macOS, replace $(uname) with OSX
+```
+To only run enabled tests for the platform the tests were built for:
 ```sh
-./src/coreclr/src/pal/tests/palsuite/runpaltests.sh $(pwd)/artifacts/obj/coreclr/$(uname).x64.Debug
+artifacts/bin/coreclr/$(uname).x64.Debug/paltests/runpaltests.sh $(pwd)/artifacts/bin/coreclr/$(uname).x64.Debug/paltests
+# on macOS, replace $(uname) with OSX
 ```
 
 Test results will go into: `/tmp/PalTestOutput/default/pal_tests.xml`
+
+To disable tests in the CI edit
+`src/coreclr/src/pal/tests/palsuite/issues.targets`

eng/Subsets.props

@@ -67,6 +67,7 @@
   </PropertyGroup>
 
   <PropertyGroup>
+    <_subset>$(_subset.Replace('+clr.paltests+', '+clr.paltests+clr.paltestlist+'))</_subset>
     <_subset>$(_subset.Replace('+clr+', '+$(DefaultCoreClrSubsets)+'))</_subset>
     <_subset>$(_subset.Replace('+mono+', '+$(DefaultMonoSubsets)+'))</_subset>
     <_subset>$(_subset.Replace('+libs+', '+$(DefaultLibrariesSubsets)+'))</_subset>
@@ -84,6 +85,8 @@
     <!-- CoreClr -->
     <SubsetName Include="Clr" Description="The CoreCLR runtime, LinuxDac, CoreLib (+ native), tools and packages." />
     <SubsetName Include="Clr.Runtime" Description="The CoreCLR .NET runtime." />
+    <SubsetName Include="Clr.PalTests" Description="The CoreCLR PAL tests." />
+    <SubsetName Include="Clr.PalTestList" Description="Generate the list of the CoreCLR PAL tests. When using the command line, use Clr.PalTests instead." />
     <SubsetName Include="Clr.Jit" Description="The JIT for the CoreCLR .NET runtime." />
     <SubsetName Include="Clr.AllJits" Description="All of the cross-targeting JIT compilers for the CoreCLR .NET runtime." />
     <SubsetName Include="Clr.CoreLib" Description="The managed System.Private.CoreLib library for CoreCLR." />
@@ -147,6 +150,10 @@
     <ClrRuntimeBuildSubsets>$(ClrRuntimeBuildSubsets);ClrJitSubset=true</ClrRuntimeBuildSubsets>
   </PropertyGroup>
 
+  <PropertyGroup Condition="$(_subset.Contains('+clr.paltests+'))">
+    <ClrRuntimeBuildSubsets>$(ClrRuntimeBuildSubsets);ClrPalTestsSubset=true</ClrRuntimeBuildSubsets>
+  </PropertyGroup>
+
   <PropertyGroup Condition="$(_subset.Contains('+clr.alljits+'))">
     <ClrRuntimeBuildSubsets>$(ClrRuntimeBuildSubsets);ClrAllJitsSubset=true;ClrJitSubset=true</ClrRuntimeBuildSubsets>
   </PropertyGroup>
@@ -155,6 +162,10 @@
     <ProjectToBuild Include="$(CoreClrProjectRoot)runtime.proj" AdditionalProperties="%(AdditionalProperties);$(ClrRuntimeBuildSubsets)" Category="clr" />
   </ItemGroup>
 
+  <ItemGroup Condition="$(_subset.Contains('+clr.paltestlist+'))">
+    <ProjectToBuild Include="$(CoreClrProjectRoot)src/pal/tests/palsuite/producepaltestlist.proj" />
+  </ItemGroup>
+
   <ItemGroup Condition="$(_subset.Contains('+linuxdac+')) and $([MSBuild]::IsOsPlatform(Windows)) and '$(TargetArchitecture)' != 'x86'">
     <ProjectToBuild Include="$(CoreClrProjectRoot)runtime.proj" AdditionalProperties="%(AdditionalProperties);$(ClrDefaultRuntimeBuildSubsets);CrossDac=linux" Category="clr" />
   </ItemGroup>

eng/native/functions.cmake

@@ -356,20 +356,19 @@ function(install_symbols symbol_file destination_path)
   endif()
 endfunction()
 
-# install_clr(TARGETS TARGETS targetName [targetName2 ...] [ADDITIONAL_DESTINATION destination])
+# install_clr(TARGETS TARGETS targetName [targetName2 ...] [ADDITIONAL_DESTINATIONS destination])
 function(install_clr)
-  set(oneValueArgs ADDITIONAL_DESTINATION)
-  set(multiValueArgs TARGETS)
-  cmake_parse_arguments(INSTALL_CLR "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGV})
+  set(multiValueArgs TARGETS ADDITIONAL_DESTINATIONS)
+  cmake_parse_arguments(INSTALL_CLR "" "" "${multiValueArgs}" ${ARGV})
 
   if ("${INSTALL_CLR_TARGETS}" STREQUAL "")
     message(FATAL_ERROR "At least one target must be passed to install_clr(TARGETS )")
   endif()
 
   set(destinations ".")
 
-  if (NOT "${INSTALL_CLR_ADDITIONAL_DESTINATION}" STREQUAL "")
-    list(APPEND destinations ${INSTALL_CLR_ADDITIONAL_DESTINATION})
+  if (NOT "${INSTALL_CLR_ADDITIONAL_DESTINATIONS}" STREQUAL "")
+    list(APPEND destinations ${INSTALL_CLR_ADDITIONAL_DESTINATIONS})
   endif()
 
   foreach(targetName ${INSTALL_CLR_TARGETS})

eng/pipelines/common/templates/runtimes/run-test-job.yml

@@ -313,6 +313,9 @@ jobs:
 
         runCrossGen: ${{ and(eq(parameters.readyToRun, true), ne(parameters.crossgen2, true)) }}
         runCrossGen2: ${{ and(eq(parameters.readyToRun, true), eq(parameters.crossgen2, true)) }}
+        ${{ if and(ne(parameters.testGroup, 'innerloop'), eq(parameters.runtimeFlavor, 'coreclr')) }}:
+          runPALTestsDir: '$(coreClrProductRootFolderPath)/paltests'
+
         compositeBuildMode: ${{ parameters.compositeBuildMode }}
         runInUnloadableContext: ${{ parameters.runInUnloadableContext }}
 

eng/pipelines/common/templates/runtimes/send-to-helix-step.yml

@@ -17,6 +17,7 @@ parameters:
   timeoutPerTestInMinutes: ''
   runCrossGen: ''
   runCrossGen2: ''
+  runPALTestsDir: ''
   compositeBuildMode: false
   helixProjectArguments: ''
   runInUnloadableContext: ''
@@ -48,6 +49,7 @@ steps:
       _LongRunningGcTests: ${{ parameters.longRunningGcTests }}
       _GcSimulatorTests: ${{ parameters.gcSimulatorTests }}
       _Scenarios: ${{ join(',', parameters.scenarios) }}
+      _PALTestsDir: ${{ parameters.runPALTestsDir }}
       _TimeoutPerTestCollectionInMinutes: ${{ parameters.timeoutPerTestCollectionInMinutes }}
       _TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }}
       runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }}

eng/pipelines/coreclr/templates/build-job.yml

@@ -109,6 +109,12 @@ jobs:
     - ${{ if eq(parameters.testGroup, 'clrinterpreter') }}:
       - name: clrInterpreterBuildArg
         value: '-cmakeargs "-DFEATURE_INTERPRETER=1"'
+
+    - name: clrBuildPALTestsBuildArg
+      value: ''
+    - ${{ if ne(parameters.testGroup, 'innerloop') }}:
+      - name: clrBuildPALTestsBuildArg
+        value: '-paltests '
 
     - ${{ parameters.variables }}
 
@@ -141,7 +147,7 @@ jobs:
 
     # Build CoreCLR Runtime
     - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
-      - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) $(osArg) -ci $(compilerArg) $(officialBuildIdArg) $(clrInterpreterBuildArg)
+      - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) $(osArg) -ci $(compilerArg) $(clrBuildPALTestsBuildArg) $(officialBuildIdArg) $(clrInterpreterBuildArg)
         displayName: Build CoreCLR Runtime
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
       - script: set __TestIntermediateDir=int&&$(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci $(enforcePgoArg) $(officialBuildIdArg) $(clrInterpreterBuildArg)
@@ -154,7 +160,7 @@ jobs:
         displayName: Disk Usage after Build
 
     # Build CoreCLR Managed Components
-    - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib+clr.nativecorelib+clr.tools+clr.packages $(crossArg) -arch $(archType) $(osArg) -c $(buildConfig) $(officialBuildIdArg) -ci
+    - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib+clr.nativecorelib+clr.tools+clr.packages+clr.paltestlist $(crossArg) -arch $(archType) $(osArg) -c $(buildConfig) $(officialBuildIdArg) -ci
       displayName: Build managed product components and packages
 
     # Run CoreCLR Tools unit tests

src/coreclr/build-runtime.cmd

@@ -84,6 +84,7 @@ set __SkipCrossArchNative=0
 set __SkipGenerateVersion=0
 set __RestoreOptData=1
 set __BuildJit=1
+set __BuildPALTests=0
 set __BuildAllJits=1
 set __BuildRuntime=1
 set __CrossArch=

src/coreclr/build-runtime.sh

@@ -25,6 +25,7 @@ usage_list+=("-staticanalyzer: skip native image generation.")
 usage_list+=("-skipjit: skip building jit.")
 usage_list+=("-skipalljits: skip building crosstargetting jits.")
 usage_list+=("-skipruntime: skip building runtime.")
+usage_list+=("-paltests: build the pal tests.")
 
 setup_dirs_local()
 {
@@ -135,6 +136,10 @@ handle_arguments_local() {
         skipruntime|-skipruntime)
             __BuildRuntime=0
             ;;
+
+        paltests|-paltests)
+            __BuildPALTests=1
+            ;;
         *)
             __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
             ;;
@@ -187,7 +192,7 @@ __UseNinja=0
 __VerboseBuild=0
 __ValidateCrossArg=1
 __CMakeArgs=""
-__BuildJit=1
+__BuildPALTests=0
 __BuildAllJits=1
 __BuildRuntime=1
 
@@ -247,6 +252,7 @@ restore_optdata
 # Build the coreclr (native) components.
 __CMakeArgs="-DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument -DCLR_CMAKE_OPTDATA_PATH=$__PgoOptDataPath -DCLR_CMAKE_PGO_OPTIMIZE=$__PgoOptimize -DCLR_REPO_ROOT_DIR=\"$__RepoRootDir\" $__CMakeArgs"
 __CMakeArgs="-DCLR_CMAKE_BUILD_SUBSET_JIT=$__BuildJit -DCLR_CMAKE_BUILD_SUBSET_ALLJITS=$__BuildAllJits -DCLR_CMAKE_BUILD_SUBSET_RUNTIME=$__BuildRuntime $__CMakeArgs"
+__CMakeArgs="-DCLR_CMAKE_BUILD_TESTS=$__BuildPALTests $__CMakeArgs"
 
 if [[ "$__SkipConfigure" == 0 && "$__CodeCoverage" == 1 ]]; then
     __CMakeArgs="-DCLR_CMAKE_ENABLE_CODE_COVERAGE=1 $__CMakeArgs"

src/coreclr/runtime.proj

@@ -23,6 +23,7 @@
       <_CoreClrBuildArg Condition="'$(NoPgoOptimize)' == 'true'" Include="-nopgooptimize" />
       <_CoreClrBuildArg Condition="'$(ClrRuntimeSubset)' != 'true'" Include="-skipruntime" />
       <_CoreClrBuildArg Condition="'$(ClrJitSubset)' != 'true'" Include="-skipjit" />
+      <_CoreClrBuildArg Condition="'$(ClrPalTestsSubset)' == 'true'" Include="-paltests" />
       <_CoreClrBuildArg Condition="'$(ClrAllJitsSubset)' != 'true'" Include="-skipalljits" />
       <_CoreClrBuildArg Condition="'$(NoPgoOptimize)' == 'true'" Include="-nopgooptimize" />
     </ItemGroup>

src/coreclr/src/debug/createdump/CMakeLists.txt

@@ -86,4 +86,4 @@ endif(CLR_CMAKE_HOST_OSX)
 
 endif(CLR_CMAKE_HOST_WIN32)
 
-install_clr(TARGETS createdump ADDITIONAL_DESTINATION sharedFramework)
+install_clr(TARGETS createdump ADDITIONAL_DESTINATIONS sharedFramework)

src/coreclr/src/dlls/clretwrc/CMakeLists.txt

@@ -20,6 +20,6 @@ add_library_clr(clretwrc SHARED
 )
 
 # add the install targets
-install_clr(TARGETS clretwrc ADDITIONAL_DESTINATION sharedFramework)
+install_clr(TARGETS clretwrc ADDITIONAL_DESTINATIONS sharedFramework)
 
 add_dependencies(clretwrc eventing_headers)

src/coreclr/src/dlls/dbgshim/CMakeLists.txt

@@ -78,4 +78,4 @@ endif(CLR_CMAKE_HOST_WIN32)
 target_link_libraries(dbgshim ${DBGSHIM_LIBRARIES})
 
 # add the install targets
-install_clr(TARGETS dbgshim ADDITIONAL_DESTINATION sharedFramework)
+install_clr(TARGETS dbgshim ADDITIONAL_DESTINATIONS sharedFramework)

src/coreclr/src/dlls/mscordac/CMakeLists.txt

@@ -194,7 +194,7 @@ if(FEATURE_SINGLE_FILE_DIAGNOSTICS)
 endif(FEATURE_SINGLE_FILE_DIAGNOSTICS)
 
 # add the install targets
-install_clr(TARGETS mscordaccore ADDITIONAL_DESTINATION sharedFramework)
+install_clr(TARGETS mscordaccore ADDITIONAL_DESTINATIONS sharedFramework)
 
 if(CLR_CMAKE_HOST_WIN32)
     set(LONG_NAME_HOST_ARCH ${CLR_CMAKE_HOST_ARCH})

src/coreclr/src/dlls/mscordbi/CMakeLists.txt

@@ -118,4 +118,4 @@ if(FEATURE_SINGLE_FILE_DIAGNOSTICS)
 endif(FEATURE_SINGLE_FILE_DIAGNOSTICS)
 
 # add the install targets
-install_clr(TARGETS mscordbi ADDITIONAL_DESTINATION sharedFramework)
+install_clr(TARGETS mscordbi ADDITIONAL_DESTINATIONS sharedFramework)

src/coreclr/src/dlls/mscoree/coreclr/CMakeLists.txt

@@ -228,7 +228,7 @@ if(CLR_CMAKE_TARGET_WIN32)
 endif(CLR_CMAKE_TARGET_WIN32)
 
 # add the install targets
-install_clr(TARGETS coreclr ADDITIONAL_DESTINATION sharedFramework)
+install_clr(TARGETS coreclr ADDITIONAL_DESTINATIONS sharedFramework)
 
 # publish coreclr_static lib
 _install(TARGETS coreclr_static DESTINATION lib)

src/coreclr/src/dlls/mscorrc/CMakeLists.txt

@@ -15,7 +15,7 @@ if(CLR_CMAKE_HOST_WIN32)
         include.rc
     )
 
-    install_clr(TARGETS mscorrc ADDITIONAL_DESTINATION sharedFramework)
+    install_clr(TARGETS mscorrc ADDITIONAL_DESTINATIONS sharedFramework)
 else()
     build_resources(${CMAKE_CURRENT_SOURCE_DIR}/include.rc mscorrc TARGET_CPP_FILE)
 

src/coreclr/src/jit/CMakeLists.txt

@@ -12,9 +12,10 @@ add_compile_options($<$<CXX_COMPILER_ID:MSVC>:-W4>)
 
 function(create_standalone_jit)
 
-  set(oneValueArgs TARGET OS ARCH ADDITIONAL_DESTINATION)
+  set(oneValueArgs TARGET OS ARCH)
+  set(multiValueArgs ADDITIONAL_DESTINATIONS)
   set(options NOALTJIT)
-  cmake_parse_arguments(TARGETDETAILS "${options}" "${oneValueArgs}" "" ${ARGN})
+  cmake_parse_arguments(TARGETDETAILS "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
 
   set(JIT_ARCH_LINK_LIBRARIES gcinfo_${TARGETDETAILS_OS}_${TARGETDETAILS_ARCH})
 
@@ -30,10 +31,10 @@ function(create_standalone_jit)
     clr_unknown_arch()
   endif()
 
-  if (TARGETDETAILS_ADDITIONAL_DESTINATION STREQUAL "")
+  if (TARGETDETAILS_ADDITIONAL_DESTINATIONS STREQUAL "")
     add_jit(${TARGETDETAILS_TARGET})
   else()
-    add_jit(${TARGETDETAILS_TARGET} ADDITIONAL_DESTINATION "${TARGETDETAILS_ADDITIONAL_DESTINATION}")
+    add_jit(${TARGETDETAILS_TARGET} ADDITIONAL_DESTINATIONS "${TARGETDETAILS_ADDITIONAL_DESTINATIONS}")
   endif()
 
   set_target_definitions_to_custom_os_and_arch(${ARGN})
@@ -411,7 +412,7 @@ else()
 endif(CLR_CMAKE_HOST_UNIX)
 
 # Shared function for generating JIT
-# optional arguments: ADDITIONAL_DESTINATION path
+# optional arguments: ADDITIONAL_DESTINATIONS path
 function(add_jit jitName)
 
     set_source_files_properties(${JIT_EXPORTS_FILE} PROPERTIES GENERATED TRUE)
@@ -459,7 +460,7 @@ else()
 set(TARGET_OS_NAME win)
 endif()
 
-create_standalone_jit(TARGET clrjit OS ${TARGET_OS_NAME} ARCH ${ARCH_TARGET_NAME} NOALTJIT ADDITIONAL_DESTINATION sharedFramework)
+create_standalone_jit(TARGET clrjit OS ${TARGET_OS_NAME} ARCH ${ARCH_TARGET_NAME} NOALTJIT ADDITIONAL_DESTINATIONS sharedFramework)
 
 # Enable profile guided optimization
 add_pgo(clrjit)

src/coreclr/src/pal/src/eventprovider/lttngprovider/CMakeLists.txt

@@ -72,4 +72,4 @@ set_target_properties(coreclrtraceptprovider PROPERTIES LINKER_LANGUAGE CXX)
 # Install the static eventprovider library
 _install(TARGETS eventprovider DESTINATION lib)
 # Install the static coreclrtraceptprovider library
-install_clr(TARGETS coreclrtraceptprovider ADDITIONAL_DESTINATION sharedFramework)
+install_clr(TARGETS coreclrtraceptprovider ADDITIONAL_DESTINATIONS sharedFramework paltests)