Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 43 additions & 20 deletions eng/pipelines/common/templates/runtimes/build-test-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ parameters:
container: ''
testGroup: ''
liveRuntimeBuildConfig: ''
testBuildPhase: ''

# When set to a non-empty value (Debug / Release), it determines libraries
# build configuration to use for the tests. Setting this property implies
Expand Down Expand Up @@ -50,13 +51,24 @@ jobs:
${{ if eq(variables['System.TeamProject'], 'internal') }}:
continueOnError: true

# Compute job name from template parameters
${{ if eq(parameters.testGroup, 'innerloop') }}:
name: '${{ parameters.runtimeFlavor }}_common_test_build_p0_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'
displayName: '${{ parameters.runtimeFlavorDisplayName }} Common Pri0 Test Build ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
${{ if ne(parameters.testBuildPhase, 'targetGeneric') }}:
name: '${{ parameters.runtimeFlavor }}_common_test_build_p0_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'
displayName: '${{ parameters.runtimeFlavorDisplayName }} Common Pri0 Test Build ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
${{ if ne(parameters.testGroup, 'innerloop') }}:
${{ if ne(parameters.testBuildPhase, 'targetGeneric') }}:
name: '${{ parameters.runtimeFlavor }}_common_test_build_p1_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'
displayName: '${{ parameters.runtimeFlavorDisplayName }} Common Pri1 Test Build ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'

${{ if eq(parameters.testGroup, 'innerloop') }}:
${{ if eq(parameters.testBuildPhase, 'targetGeneric') }}:
name: '${{ parameters.runtimeFlavor }}_common_test_build_p0_AnyOS_AnyCPU_${{ parameters.buildConfig }}'
displayName: '${{ parameters.runtimeFlavorDisplayName }} Common Pri0 Test Build AnyOS AnyCPU ${{ parameters.buildConfig }}'
${{ if ne(parameters.testGroup, 'innerloop') }}:
name: '${{ parameters.runtimeFlavor }}_common_test_build_p1_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'
displayName: '${{ parameters.runtimeFlavorDisplayName }} Common Pri1 Test Build ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
${{ if eq(parameters.testBuildPhase, 'targetGeneric') }}:
name: '${{ parameters.runtimeFlavor }}_common_test_build_p1_AnyOS_AnyCPU_${{ parameters.buildConfig }}'
displayName: '${{ parameters.runtimeFlavorDisplayName }} Common Pri1 Test Build AnyOS AnyCPU ${{ parameters.buildConfig }}'

# Since the condition is being altered, merge the default with the additional conditions.
# See https://docs.microsoft.com/azure/devops/pipelines/process/conditions
Expand Down Expand Up @@ -120,7 +132,7 @@ jobs:


# Build managed test components
- script: $(coreClrRepoRootDir)build-test$(scriptExt) skipnative skipgeneratelayout skiptestwrappers $(buildConfig) $(archType) $(crossArg) $(priorityArg) ci $(librariesOverrideArg)
- script: $(coreClrRepoRootDir)build-test$(scriptExt) ${{ parameters.testBuildPhase }} skipnative skipgeneratelayout skiptestwrappers $(buildConfig) $(archType) $(crossArg) $(priorityArg) ci $(librariesOverrideArg)
displayName: Build managed test components


Expand All @@ -129,31 +141,42 @@ jobs:
parameters:
rootFolder: $(managedTestArtifactRootFolderPath)
includeRootFolder: false
archiveType: $(archiveType)
tarCompression: $(tarCompression)
archiveExtension: $(archiveExtension)
artifactName: $(managedTestArtifactName)
displayName: 'managed test components'
${{ if ne(parameters.testBuildPhase, 'targetGeneric') }}:
archiveExtension: $(archiveExtension)
archiveType: $(archiveType)
tarCompression: $(tarCompression)
artifactName: $(managedTestArtifactName)
displayName: 'managed test components'
${{ if eq(parameters.testBuildPhase, 'targetGeneric') }}:
archiveExtension: '.tar.gz'
archiveType: tar
tarCompression: gz
artifactName: $(managedGenericTestArtifactName)
displayName: 'managed test components (generic)'


# Publish .packages/microsoft.net.sdk.il needed for traversing
# test projects during the copynativeonly command in run test job.
- template: /eng/pipelines/common/upload-artifact-step.yml
parameters:
rootFolder: $(microsoftNetSdkIlFolderPath)
includeRootFolder: false
archiveType: $(archiveType)
tarCompression: $(tarCompression)
archiveExtension: $(archiveExtension)
artifactName: $(microsoftNetSdkIlArtifactName)
displayName: 'Microsoft.NET.Sdk.IL package'
- ${{ if ne(parameters.testBuildPhase, 'targetGeneric') }}:
- template: /eng/pipelines/common/upload-artifact-step.yml
parameters:
rootFolder: $(microsoftNetSdkIlFolderPath)
includeRootFolder: false
archiveType: $(archiveType)
tarCompression: $(tarCompression)
archiveExtension: $(archiveExtension)
artifactName: $(microsoftNetSdkIlArtifactName)
displayName: 'Microsoft.NET.Sdk.IL package'


# Publish Logs
- task: PublishPipelineArtifact@1
displayName: Publish Logs
inputs:
targetPath: $(Build.SourcesDirectory)/artifacts/log
artifactName: '${{ parameters.runtimeFlavor }}_Common_Runtime_TestBuildLogs_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_${{ parameters.testGroup }}'
${{ if ne(parameters.testBuildPhase, 'targetGeneric') }}:
artifactName: '${{ parameters.runtimeFlavor }}_Common_Runtime_TestBuildLogs_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_${{ parameters.testGroup }}'
${{ if eq(parameters.testBuildPhase, 'targetGeneric') }}:
artifactName: '${{ parameters.runtimeFlavor }}_Common_Runtime_TestBuildLogs_AnyOS_AnyCPU_$(buildConfig)_Lib${{ parameters.liveLibrariesBuildConfig }}_${{ parameters.testGroup }}'
continueOnError: true
condition: always()
20 changes: 20 additions & 0 deletions eng/pipelines/common/templates/runtimes/run-test-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@ parameters:
variables: {}
pool: ''
runtimeFlavorDisplayName: 'CoreCLR'
# If true, tests were built in two phases
# We will depend on both the TargetGeneric tests which are for AnyOS AnyCPU
# as well as the TargetSpecific test. Both sets of tests will be run
testBuildPhased : false

### Test run job

Expand Down Expand Up @@ -57,6 +61,12 @@ jobs:
- '${{ parameters.runtimeFlavor }}_common_test_build_p0_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{parameters.buildConfig }}'
- ${{ if ne(parameters.testGroup, 'innerloop') }}:
- '${{ parameters.runtimeFlavor }}_common_test_build_p1_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{parameters.buildConfig }}'
- ${{ if eq(parameters.testBuildPhased, true) }}:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we have to still depend on the non-phased build if this is to true?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

testBuildPhased here means that the tests were built in two phases. TargetGeneric and TargetSpecific tests. We join them here by unzipping into the same directory.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So if testBuildPhased==true we still depend on:
- '${{ parameters.runtimeFlavor }}_common_test_build_p0_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{parameters.buildConfig }}'

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@trylek And I discussed this briefly during the original review.

We may eventually refactor and build the TargetSpecific tests as part of run-test-job. He was suggesting collecting stats to determine whether the extra build legs added value or not.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That sounds like a reasonable, interesting idea.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In fact, I was mostly worried (based on earlier discussions with @jashook) that adding too many legs can be counterproductive by increasing the load on machine allocation and provisioning. I however think that having the specific test build legs initially separate is probably a good idea to get an initial reading on the perf characteristics.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right, but if we refactor and just have build-test legs for the generic tests and then the target specific builds are done at the run-test-job, before sending to helix, we would be reducing the number of legs we will have by the number of target specific tests we currently have, so I think it would improve perf and reliability.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I suspect you are right.

  • We have reduce the target specific build by at least 90%.
  • The extra leg appears expensive (maybe 50% overhead). I was seeing target specific 10 minute builds in the pipelines. I don't think it was that long on my local machine
  • We should only have to rerun a given leg < 1% of the time (if we achieve 95% success).
  • Fewer legs should reduce infra burden, so we could se benefits elsewhere.
  • Seems like the savings could easily amortize the cost.

Filtering to build the TargetSpecific tests is currently slow. But if we make msbuild improvements this may go away.

# Also depend on AnyOS AnyCPU tests
- ${{ if eq(parameters.testGroup, 'innerloop') }}:
- '${{ parameters.runtimeFlavor }}_common_test_build_p0_AnyOS_AnyCPU_${{parameters.buildConfig }}'
- ${{ if ne(parameters.testGroup, 'innerloop') }}:
- '${{ parameters.runtimeFlavor }}_common_test_build_p1_AnyOS_AnyCPU_${{parameters.buildConfig }}'
- ${{ if ne(parameters.stagedBuild, true) }}:
- ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
- ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
Expand Down Expand Up @@ -189,6 +199,16 @@ jobs:

# Download and unzip managed test artifacts
- ${{ if ne(parameters.corefxTests, true) }}:
# First download and unzip AnyOS AnyCPU test (if needed)
- ${{ if eq(parameters.testBuildPhased, true) }}:
- template: /eng/pipelines/common/download-artifact-step.yml
parameters:
unpackFolder: '$(managedTestArtifactRootFolderPath)'
artifactFileName: '$(managedGenericTestArtifactName).tar.gz'
artifactName: '$(managedGenericTestArtifactName)'
displayName: 'generic managed test artifacts'
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This might be worth 1-2 more comments explaining the generic vs. specific test management. It would be probably too nitpicky to alternate the displayName of the pre-existing download between "managed test artifacts" and "generic managed test artifacts" but we should try to keep these scripts easy to understand and possibly refactor.


# Download and unzip target specific tests
- template: /eng/pipelines/common/download-artifact-step.yml
parameters:
unpackFolder: '$(managedTestArtifactRootFolderPath)'
Expand Down
14 changes: 14 additions & 0 deletions eng/pipelines/coreclr/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,17 @@ jobs:
#
# Checked test builds
#
- template: /eng/pipelines/common/platform-matrix.yml
parameters:
jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml
buildConfig: checked
platforms:
- OSX_x64
testGroup: outerloop
jobParameters:
liveLibrariesBuildConfig: Release
testBuildPhase: targetGeneric

- template: /eng/pipelines/common/platform-matrix.yml
parameters:
jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml
Expand All @@ -107,6 +118,7 @@ jobs:
testGroup: outerloop
jobParameters:
liveLibrariesBuildConfig: Release
testBuildPhase: targetSpecific

#
# Checked JIT test runs
Expand All @@ -121,6 +133,7 @@ jobs:
jobParameters:
testGroup: outerloop
liveLibrariesBuildConfig: Release
testBuildPhased: true

#
# Checked R2R test runs
Expand All @@ -146,6 +159,7 @@ jobs:
readyToRun: true
displayNameArgs: R2R
liveLibrariesBuildConfig: Release
testBuildPhased: true

#
# Crossgen-comparison jobs
Expand Down
3 changes: 3 additions & 0 deletions eng/pipelines/coreclr/templates/xplat-pipeline-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,9 @@ jobs:
- name: corelibProductArtifactName
value: 'CoreLib_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'

- name: managedGenericTestArtifactName
value: 'CoreCLRManagedTestArtifacts_AnyOS_AnyCPU_$(buildConfig)'

- name: managedTestArtifactName
value: 'CoreCLRManagedTestArtifacts_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_$(archType)_$(buildConfig)'

Expand Down
19 changes: 19 additions & 0 deletions eng/pipelines/runtime.yml
Original file line number Diff line number Diff line change
Expand Up @@ -576,6 +576,21 @@ jobs:
# CoreCLR Test builds using live libraries release build
# Only when CoreCLR is changed
#
- template: /eng/pipelines/common/platform-matrix.yml
parameters:
jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml
buildConfig: checked
platforms:
- OSX_x64
jobParameters:
testGroup: innerloop
liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
testBuildPhase: targetGeneric
condition: >-
or(
eq(dependencies.checkout.outputs['SetPathVars_coreclr.containsChange'], true),
eq(variables['isFullMatrix'], true))

- template: /eng/pipelines/common/platform-matrix.yml
parameters:
jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml
Expand All @@ -588,6 +603,7 @@ jobs:
jobParameters:
testGroup: innerloop
liveLibrariesBuildConfig: Release
testBuildPhase: targetSpecific
condition: >-
or(
eq(dependencies.checkout.outputs['SetPathVars_coreclr.containsChange'], true),
Expand All @@ -609,6 +625,7 @@ jobs:
jobParameters:
testGroup: innerloop
liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
testBuildPhase: targetSpecific
condition: >-
or(
eq(dependencies.checkout.outputs['SetPathVars_coreclr.containsChange'], true),
Expand All @@ -632,6 +649,7 @@ jobs:
jobParameters:
testGroup: innerloop
liveLibrariesBuildConfig: Release
testBuildPhased: true
condition: >-
or(
eq(dependencies.checkout.outputs['SetPathVars_coreclr.containsChange'], true),
Expand All @@ -651,6 +669,7 @@ jobs:
jobParameters:
testGroup: innerloop
liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
testBuildPhased: true
condition: >-
or(
eq(dependencies.checkout.outputs['SetPathVars_coreclr.containsChange'], true),
Expand Down
6 changes: 6 additions & 0 deletions src/coreclr/tests/src/Interop/COM/Activator/Activator.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -32,4 +32,10 @@
</ItemGroup>
<Move SourceFiles="@(ServerAssembly)" DestinationFolder="$(ServerAssemblyDest)" />
</Target>
<PropertyGroup>
<CLRTestNeedTarget>1</CLRTestNeedTarget>
</PropertyGroup>
<ItemGroup>
<TraitTags Include="OsSpecific" />
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,10 @@
<ProjectReference Include="AssemblyC.csproj" />
<ProjectReference Include="AssemblyContracts.csproj" />
</ItemGroup>
<PropertyGroup>
<CLRTestNeedTarget>1</CLRTestNeedTarget>
</PropertyGroup>
<ItemGroup>
<TraitTags Include="OsSpecific" />
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,10 @@
<ProjectReference Include="AssemblyC.csproj" />
<ProjectReference Include="AssemblyContracts.csproj" />
</ItemGroup>
<PropertyGroup>
<CLRTestNeedTarget>1</CLRTestNeedTarget>
</PropertyGroup>
<ItemGroup>
<TraitTags Include="OsSpecific" />
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,10 @@
<ItemGroup>
<Compile Include="AssemblyC.cs" />
</ItemGroup>
<PropertyGroup>
<CLRTestNeedTarget>1</CLRTestNeedTarget>
</PropertyGroup>
<ItemGroup>
<TraitTags Include="OsSpecific" />
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,10 @@
<ItemGroup>
<Compile Include="AssemblyContracts.cs" />
</ItemGroup>
<PropertyGroup>
<CLRTestNeedTarget>1</CLRTestNeedTarget>
</PropertyGroup>
<ItemGroup>
<TraitTags Include="OsSpecific" />
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,10 @@
<ItemGroup>
<ProjectReference Include="../MockReferenceTrackerRuntime/CMakeLists.txt" />
</ItemGroup>
<PropertyGroup>
<CLRTestNeedTarget>1</CLRTestNeedTarget>
</PropertyGroup>
<ItemGroup>
<TraitTags Include="OsSpecific" />
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -34,4 +34,10 @@
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
<PropertyGroup>
<CLRTestNeedTarget>1</CLRTestNeedTarget>
</PropertyGroup>
<ItemGroup>
<TraitTags Include="OsSpecific" />
</ItemGroup>
</Project>
6 changes: 6 additions & 0 deletions src/coreclr/tests/src/Interop/COM/Dynamic/Dynamic.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,10 @@
<ProjectReference Include="Server/CMakeLists.txt" />
<ProjectReference Include="$(TestSourceDir)Common/CoreCLRTestLibrary/CoreCLRTestLibrary.csproj" />
</ItemGroup>
<PropertyGroup>
<CLRTestNeedTarget>1</CLRTestNeedTarget>
</PropertyGroup>
<ItemGroup>
<TraitTags Include="OsSpecific" />
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,10 @@
<ProjectReference Include="../../NativeServer/CMakeLists.txt" />
<ProjectReference Include="$(TestSourceDir)Common/CoreCLRTestLibrary/CoreCLRTestLibrary.csproj" />
</ItemGroup>
<PropertyGroup>
<CLRTestNeedTarget>1</CLRTestNeedTarget>
</PropertyGroup>
<ItemGroup>
<TraitTags Include="OsSpecific" />
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -31,4 +31,10 @@
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
<PropertyGroup>
<CLRTestNeedTarget>1</CLRTestNeedTarget>
</PropertyGroup>
<ItemGroup>
<TraitTags Include="OsSpecific" />
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -21,4 +21,10 @@
<ProjectReference Include="../../NativeServer/CMakeLists.txt" />
<ProjectReference Include="$(TestSourceDir)Common/CoreCLRTestLibrary/CoreCLRTestLibrary.csproj" />
</ItemGroup>
<PropertyGroup>
<CLRTestNeedTarget>1</CLRTestNeedTarget>
</PropertyGroup>
<ItemGroup>
<TraitTags Include="OsSpecific" />
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,10 @@
<ProjectReference Include="../../NativeServer/CMakeLists.txt" />
<ProjectReference Include="$(TestSourceDir)Common/CoreCLRTestLibrary/CoreCLRTestLibrary.csproj" />
</ItemGroup>
<PropertyGroup>
<CLRTestNeedTarget>1</CLRTestNeedTarget>
</PropertyGroup>
<ItemGroup>
<TraitTags Include="OsSpecific" />
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,10 @@
<ProjectReference Include="../../NativeServer/CMakeLists.txt" />
<ProjectReference Include="$(TestSourceDir)Common/CoreCLRTestLibrary/CoreCLRTestLibrary.csproj" />
</ItemGroup>
<PropertyGroup>
<CLRTestNeedTarget>1</CLRTestNeedTarget>
</PropertyGroup>
<ItemGroup>
<TraitTags Include="OsSpecific" />
</ItemGroup>
</Project>
Loading