Run nativeaot tests in helix

eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml

@@ -2,14 +2,15 @@ parameters:
   buildConfig: ''
   osSubgroup: ''
   container: ''
-  testGroup: ''
+  testBuildArgs: ''
   crossBuild: false
   readyToRun: false
   liveLibrariesBuildConfig: ''
   compositeBuildMode: false
   helixQueues: ''
   displayNameArgs: ''
   runInUnloadableContext: false
+  nativeAotTest: false
   runtimeFlavor: 'mono'
   runtimeVariant: 'monointerpreter'
   variables: {}
@@ -26,11 +27,11 @@ parameters:
 
 steps:
   - ${{ if eq(parameters.runtimeFlavor, 'mono') }}:
-    - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) /p:RuntimeVariant=${{ parameters.runtimeVariant }} /p:LibrariesConfiguration=${{ parameters.buildConfig }} -mono -ci os ${{ parameters.osGroup }} ${{ parameters.archType }} $(buildConfigUpper)
+    - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) /p:RuntimeVariant=${{ parameters.runtimeVariant }} ${{ parameters.testBuildArgs }} -mono os ${{ parameters.osGroup }} ${{ parameters.archType }} $(buildConfigUpper) /p:LibrariesConfiguration=${{ parameters.buildConfig }} -ci
       displayName: Build Tests
 
   - ${{ if ne(parameters.runtimeFlavor, 'mono') }}:
-    - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) /p:LibrariesConfiguration=${{ parameters.buildConfig }} -ci os ${{ parameters.osGroup }} ${{ parameters.archType }} $(buildConfigUpper)
+    - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) ${{ parameters.archType }} $(buildConfigUpper) ${{ parameters.testBuildArgs }} -ci
       displayName: Build Tests
 
 
@@ -104,6 +105,7 @@ steps:
         runCrossGen2: ${{ eq(parameters.readyToRun, true) }}
         compositeBuildMode: ${{ parameters.compositeBuildMode }}
         runInUnloadableContext: ${{ parameters.runInUnloadableContext }}
+        nativeAotTest: ${{ parameters.nativeAotTest }}
 
         ${{ if eq(variables['System.TeamProject'], 'internal') }}:
           # Access token variable for internal project from the

eng/pipelines/common/templates/runtimes/build-test-job.yml

@@ -98,12 +98,8 @@ jobs:
 
       # Only build GCSimulator tests when the gc-simulator group is specified.
       - ${{ if eq(parameters.testGroup, 'gc-simulator') }}:
-        - ${{ if eq(parameters.osGroup, 'windows') }}:
           - name: testTreeFilterArg
             value: 'tree GC/Scenarios/GCSimulator'
-        - ${{ if ne(parameters.osGroup, 'windows') }}:
-          - name: testTreeFilterArg
-            value: '-tree:GC/Scenarios/GCSimulator'
 
     steps:
 

eng/pipelines/common/templates/runtimes/run-test-job.yml

@@ -139,12 +139,8 @@ jobs:
 
     # Only build GCSimulator tests when the gc-simulator group is specified.
     - ${{ if eq(parameters.testGroup, 'gc-simulator') }}:
-      - ${{ if eq(parameters.osGroup, 'windows') }}:
-        - name: testTreeFilterArg
-          value: 'tree GC/Scenarios/GCSimulator'
-      - ${{ if ne(parameters.osGroup, 'windows') }}:
-        - name: testTreeFilterArg
-          value: '-tree:GC/Scenarios/GCSimulator'
+      - name: testTreeFilterArg
+        value: 'tree GC/Scenarios/GCSimulator'
 
     # Variables used for SuperPMI collection
     - ${{ if eq(parameters.SuperPmiCollect, true) }}:

eng/pipelines/common/templates/runtimes/send-to-helix-step.yml

@@ -22,6 +22,7 @@ parameters:
   helixProjectArguments: ''
   runInUnloadableContext: ''
   tieringTest: ''
+  nativeAotTest: ''
   longRunningGcTests: ''
   gcSimulatorTests: ''
   runtimeFlavor: 'CoreCLR'
@@ -51,6 +52,7 @@ steps:
       _CompositeBuildMode: ${{ parameters.compositeBuildMode }}
       _RunInUnloadableContext: ${{ parameters.runInUnloadableContext }}
       _TieringTest: ${{ parameters.tieringTest }}
+      _NativeAotTest: ${{ parameters.nativeAotTest }}
       _LongRunningGcTests: ${{ parameters.longRunningGcTests }}
       _GcSimulatorTests: ${{ parameters.gcSimulatorTests }}
       _Scenarios: ${{ join(',', parameters.scenarios) }}

eng/pipelines/coreclr/nativeaot-post-build-steps.yml

@@ -3,32 +3,25 @@ parameters:
   archType: ''
   osGroup: ''
   osSubgroup: ''
-  platform: ''
-  pgoType: ''
-  runtimeVariant: ''
-  uploadTests: false
-  testFilter: tree nativeaot
-  runSingleFileTests: true
+  testBuildArgs: ''
+  helixQueues: ''
 
 steps:
 # Can't run arm/arm64 tests on x64 build machines
 - ${{ if and(ne(parameters.archType, 'arm'), ne(parameters.archType, 'arm64')) }}:
 
-  # Build coreclr native test output
-  - ${{ if eq(parameters.osGroup, 'windows') }}:
-    - script: $(Build.SourcesDirectory)/src/tests/build.cmd nativeaot $(buildConfigUpper) ${{ parameters.archType }} ${{ parameters.testFilter }} /p:BuildNativeAotFrameworkObjects=true
-      displayName: Build tests
-  - ${{ if ne(parameters.osGroup, 'windows') }}:
-    - script: $(Build.SourcesDirectory)/src/tests/build.sh nativeaot $(buildConfigUpper) ${{ parameters.archType }} '${{ parameters.testFilter }}'
-      displayName: Build tests
-
-  - ${{ if eq(parameters.runSingleFileTests, true) }}:
-    - ${{ if eq(parameters.osGroup, 'windows') }}:
-      - script: $(Build.SourcesDirectory)/src/tests/run.cmd runnativeaottests $(buildConfigUpper) ${{ parameters.archType }}
-        displayName: Run tests in single file mode
-    - ${{ if ne(parameters.osGroup, 'windows') }}:
-      - script: $(Build.SourcesDirectory)/src/tests/run.sh --runnativeaottests $(buildConfigUpper) ${{ parameters.archType }}
-        displayName: Run tests in single file mode
+  # Build coreclr native test output and send to helix
+  - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml
+    parameters:
+      buildConfig: ${{ parameters.buildConfig }}
+      archType: ${{ parameters.archType }}
+      osGroup: ${{ parameters.osGroup }}
+      osSubgroup: ${{ parameters.osSubgroup }}
+      runtimeFlavor: coreclr
+      runtimeVariant: ''
+      testBuildArgs: ${{ parameters.testBuildArgs }}
+      nativeAotTest: true
+      helixQueues: ${{ parameters.helixQueues }}
 
   # Publishing tooling doesn't support different configs between runtime and libs, so only run tests in Release config
   - ${{ if eq(parameters.buildConfig, 'release') }}:

eng/pipelines/runtime.yml

@@ -195,16 +195,26 @@ extends:
       - template: /eng/pipelines/common/platform-matrix.yml
         parameters:
           jobTemplate: /eng/pipelines/common/global-build-job.yml
-          buildConfig: debug
+          helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
+          buildConfig: Debug
           platforms:
           - linux_x64
           - windows_x64
+          variables:
+          - name: timeoutPerTestInMinutes
+            value: 60
+          - name: timeoutPerTestCollectionInMinutes
+            value: 180
           jobParameters:
             testGroup: innerloop
             timeoutInMinutes: 120
             nameSuffix: NativeAOT
             buildArgs: -s clr.aot+host.native+libs -rc $(_BuildConfig) -lc Release -hc Release
             extraStepsTemplate: /eng/pipelines/coreclr/nativeaot-post-build-steps.yml
+            extraStepsParameters:
+              creator: dotnet-bot
+              testBuildArgs: nativeaot tree nativeaot
+            testRunNamePrefixSuffix: NativeAOT_$(_BuildConfig)
             condition: >-
               or(
                 eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true),
@@ -218,15 +228,25 @@ extends:
       - template: /eng/pipelines/common/platform-matrix.yml
         parameters:
           jobTemplate: /eng/pipelines/common/global-build-job.yml
-          buildConfig: checked
+          helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
+          buildConfig: Checked
           platforms:
           - windows_x64
+          variables:
+          - name: timeoutPerTestInMinutes
+            value: 60
+          - name: timeoutPerTestCollectionInMinutes
+            value: 180
           jobParameters:
             testGroup: innerloop
             timeoutInMinutes: 120
             nameSuffix: NativeAOT
             buildArgs: -s clr.aot+host.native+libs -rc $(_BuildConfig) -lc Release -hc Release
             extraStepsTemplate: /eng/pipelines/coreclr/nativeaot-post-build-steps.yml
+            extraStepsParameters:
+              creator: dotnet-bot
+              testBuildArgs: nativeaot tree nativeaot /p:BuildNativeAotFrameworkObjects=true
+            testRunNamePrefixSuffix: NativeAOT_$(_BuildConfig)
             condition: >-
               or(
                 eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true),
@@ -240,17 +260,27 @@ extends:
       - template: /eng/pipelines/common/platform-matrix.yml
         parameters:
           jobTemplate: /eng/pipelines/common/global-build-job.yml
-          buildConfig: release
+          helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
+          buildConfig: Release
           platforms:
           - linux_x64
           - windows_x64
           - osx_x64
+          variables:
+          - name: timeoutPerTestInMinutes
+            value: 60
+          - name: timeoutPerTestCollectionInMinutes
+            value: 180
           jobParameters:
             testGroup: innerloop
             timeoutInMinutes: 120
             nameSuffix: NativeAOT
             buildArgs: -s clr.aot+host.native+libs -rc $(_BuildConfig) -lc Release -hc Release
             extraStepsTemplate: /eng/pipelines/coreclr/nativeaot-post-build-steps.yml
+            extraStepsParameters:
+              creator: dotnet-bot
+              testBuildArgs: nativeaot tree nativeaot
+            testRunNamePrefixSuffix: NativeAOT_$(_BuildConfig)
             condition: >-
               or(
                 eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true),

src/tests/Common/helixpublishwitharcade.proj

@@ -40,6 +40,7 @@
     <_GcSimulatorTests>false</_GcSimulatorTests>
     <_RunInUnloadableContext>false</_RunInUnloadableContext>
     <_TieringTest>false</_TieringTest>
+    <_NativeAotTest>false</_NativeAotTest>
     <_TimeoutPerTestCollectionInMinutes>123</_TimeoutPerTestCollectionInMinutes>
     <_TimeoutPerTestInMinutes>234</_TimeoutPerTestInMinutes>
     <_RuntimeVariant></_RuntimeVariant>
@@ -95,6 +96,7 @@
         GcSimulatorTests=$(_GcSimulatorTests);
         RunInUnloadableContext=$(_RunInUnloadableContext);
         TieringTest=$(_TieringTest);
+        NativeAotTest=$(_NativeAotTest);
         TimeoutPerTestCollectionInMinutes=$(_TimeoutPerTestCollectionInMinutes);
         TimeoutPerTestInMinutes=$(_TimeoutPerTestInMinutes);
         RuntimeVariant=$(_RuntimeVariant);
@@ -288,6 +290,8 @@
     <Copy SourceFiles="$(MSBuildThisFileDirectory)scripts/runincontext.sh" DestinationFolder="$(CoreRootDirectory)" Condition=" '$(TestWrapperTargetsWindows)' != 'true' and '$(_RunInUnloadableContext)' == 'true'" />
     <Copy SourceFiles="$(MSBuildThisFileDirectory)scripts\tieringtest.cmd" DestinationFolder="$(CoreRootDirectory)" Condition=" '$(TestWrapperTargetsWindows)' == 'true' and '$(_TieringTest)' == 'true'" />
     <Copy SourceFiles="$(MSBuildThisFileDirectory)scripts/tieringtest.sh" DestinationFolder="$(CoreRootDirectory)" Condition=" '$(TestWrapperTargetsWindows)' != 'true' and '$(_TieringTest)' == 'true'" />
+    <Copy SourceFiles="$(MSBuildThisFileDirectory)scripts\nativeaottest.cmd" DestinationFolder="$(CoreRootDirectory)" Condition=" '$(TestWrapperTargetsWindows)' == 'true' and '$(_NativeAotTest)' == 'true'" />
+    <Copy SourceFiles="$(MSBuildThisFileDirectory)scripts/nativeaottest.sh" DestinationFolder="$(CoreRootDirectory)" Condition=" '$(TestWrapperTargetsWindows)' != 'true' and '$(_NativeAotTest)' == 'true'" />
 
     <ItemGroup Condition=" '$(SuperPmiCollect)' == 'true' ">
       <!-- We need superpmi.py and its dependencies; this is an over-approximation. The HelixCorrelationPayload directory is CoreRootDirectory, so copy the scripts there. -->
@@ -602,6 +606,7 @@
     <HelixPreCommand Include="set RunTieringTest=1" Condition=" '$(TieringTest)' == 'true' " />
     <HelixPreCommand Include="set CLRCustomTestLauncher=%HELIX_CORRELATION_PAYLOAD%\runincontext.cmd" Condition=" '$(RunInUnloadableContext)' == 'true' " />
     <HelixPreCommand Include="set CLRCustomTestLauncher=%HELIX_CORRELATION_PAYLOAD%\tieringtest.cmd" Condition=" '$(TieringTest)' == 'true' " />
+    <HelixPreCommand Include="set CLRCustomTestLauncher=%HELIX_CORRELATION_PAYLOAD%\nativeaottest.cmd" Condition=" '$(NativeAotTest)' == 'true' " />
     <HelixPreCommand Include="set __TestEnv=%HELIX_WORKITEM_PAYLOAD%\$(TestEnvFileName)" />
     <HelixPreCommand Include="set __TestTimeout=$(TimeoutPerTestInMilliseconds)" Condition=" '$(TimeoutPerTestInMilliseconds)' != '' " />
     <HelixPreCommand Include="set __CollectDumps=1" />
@@ -650,6 +655,7 @@
     <HelixPreCommand Include="export RunTieringTest=1" Condition=" '$(TieringTest)' == 'true' " />
     <HelixPreCommand Include="export CLRCustomTestLauncher=$HELIX_CORRELATION_PAYLOAD/runincontext.sh" Condition=" '$(RunInUnloadableContext)' == 'true' " />
     <HelixPreCommand Include="export CLRCustomTestLauncher=$HELIX_CORRELATION_PAYLOAD/tieringtest.sh" Condition=" '$(TieringTest)' == 'true' " />
+    <HelixPreCommand Include="export CLRCustomTestLauncher=$HELIX_CORRELATION_PAYLOAD/nativeaottest.sh" Condition=" '$(NativeAotTest)' == 'true' " />
     <HelixPreCommand Include="export __TestEnv=$HELIX_WORKITEM_PAYLOAD/$(TestEnvFileName)" />
     <HelixPreCommand Include="export __TestTimeout=$(TimeoutPerTestInMilliseconds)" Condition=" '$(TimeoutPerTestInMilliseconds)' != '' " />
     <HelixPreCommand Include="export __CollectDumps=1" />

src/tests/Common/scripts/nativeaottest.sh

@@ -11,4 +11,5 @@
 # 3. - n. Additional arguments that were passed to the test .sh
 
 exename=$(basename $2 .dll)
+chmod +x $1/native/$exename
 $_DebuggerFullPath $1/native/$exename "${@:3}"

src/tests/build.sh

@@ -234,19 +234,34 @@ handle_arguments_local() {
         test*|-test*)
             local arg="$1"
             local parts=(${arg//:/ })
-            __BuildTestProject="$__BuildTestProject${parts[1]}%3B"
+            if [[ ${#parts[@]} -eq 1 ]]; then
+                shift
+                __BuildTestProject="$__BuildTestProject$1%3B"
+            else
+                __BuildTestProject="$__BuildTestProject${parts[1]}%3B"
+            fi
             ;;
 
         dir*|-dir*)
             local arg="$1"
             local parts=(${arg//:/ })
-            __BuildTestDir="$__BuildTestDir${parts[1]}%3B"
+            if [[ ${#parts[@]} -eq 1 ]]; then
+                shift
+                __BuildTestDir="$__BuildTestDir$1%3B"
+            else
+                __BuildTestDir="$__BuildTestDir${parts[1]}%3B"
+            fi
             ;;
 
         tree*|-tree*)
             local arg="$1"
             local parts=(${arg//:/ })
-            __BuildTestTree="$__BuildTestTree${parts[1]}%3B"
+            if [[ ${#parts[@]} -eq 1 ]]; then
+                shift
+                __BuildTestTree="$__BuildTestTree$1%3B"
+            else
+                __BuildTestTree="$__BuildTestTree${parts[1]}%3B"
+            fi
             ;;
 
         runtests|-runtests)
@@ -284,7 +299,12 @@ handle_arguments_local() {
         log*|-log*)
             local arg="$1"
             local parts=(${arg//:/ })
-            __BuildLogRootName="${parts[1]}"
+            if [[ ${#parts[@]} -eq 1 ]]; then
+                shift
+                __BuildLogRootName="$1"
+            else
+                __BuildLogRootName="${parts[1]}"
+            fi
             ;;
 
         *)