From 2c6cf4d7889561ff3301bc178fcfba3e43d673b0 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Tue, 4 Jun 2024 13:54:10 -0400 Subject: [PATCH 1/3] Merge #92958 and #93082 --- eng/pipelines/global-build.yml | 2 - eng/pipelines/installer/jobs/base-job.yml | 1 - eng/pipelines/libraries/base-job.yml | 16 +-- eng/pipelines/libraries/build-job.yml | 115 +++++----------------- eng/pipelines/libraries/run-test-job.yml | 6 +- eng/pipelines/runtime-official.yml | 7 +- eng/pipelines/runtime.yml | 28 ++++-- 7 files changed, 53 insertions(+), 122 deletions(-) diff --git a/eng/pipelines/global-build.yml b/eng/pipelines/global-build.yml index 5e7cdda17d4498..aee67ca820e236 100644 --- a/eng/pipelines/global-build.yml +++ b/eng/pipelines/global-build.yml @@ -133,8 +133,6 @@ jobs: - SourceBuild_linux_x64 jobParameters: nameSuffix: PortableSourceBuild - extraStepsParameters: - name: SourceBuildPackages timeoutInMinutes: 95 condition: eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true) diff --git a/eng/pipelines/installer/jobs/base-job.yml b/eng/pipelines/installer/jobs/base-job.yml index 1075690b7a48df..ddc8d7f3dc91b1 100644 --- a/eng/pipelines/installer/jobs/base-job.yml +++ b/eng/pipelines/installer/jobs/base-job.yml @@ -12,7 +12,6 @@ parameters: container: '' buildSteps: [] dependsOn: [] - dependsOnGlobalBuild: false dependOnEvaluatePaths: false globalBuildSuffix: '' variables: [] diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index ff3f22a2eaf3a6..00adbce6bce703 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -6,7 +6,6 @@ parameters: crossBuild: false crossrootfsDir: '' framework: 'net6.0' - isOfficialAllConfigurations: false isSourceBuild: false liveRuntimeBuildConfig: '' runtimeFlavor: 'coreclr' @@ -27,12 +26,8 @@ parameters: jobs: - template: /eng/common/templates/job/job.yml parameters: - ${{ if notIn(parameters.framework, 'allConfigurations', 'net48') }}: - displayName: ${{ format('Libraries {0} {1}{2} {3} {4}', parameters.displayName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - name: ${{ format('libraries_{0}_{1}{2}_{3}_{4}', parameters.name, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - ${{ if in(parameters.framework, 'allConfigurations', 'net48') }}: - displayName: ${{ format('Libraries {0} {1} {2} {3} {4}', parameters.displayName, parameters.osGroup, parameters.framework, parameters.archType, parameters.buildConfig) }} - name: ${{ format('libraries_{0}_{1}_{2}{3}_{4}_{5}', parameters.name, parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('Libraries {0} {1}{2} {3} {4}', parameters.displayName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + name: ${{ format('libraries_{0}_{1}{2}_{3}_{4}', parameters.name, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} enableTelemetry: ${{ parameters.isOfficialBuild }} # TODO: figure out if it's needed container: ${{ parameters.container }} @@ -80,13 +75,6 @@ jobs: - _finalFrameworkArg: -framework ${{ parameters.framework }} - _extraHelixArguments: /p:BuildTargetFramework=${{ parameters.framework }} - - ${{ if eq(parameters.framework, 'allConfigurations') }}: - - _finalFrameworkArg: -allConfigurations - - _testModeArg: /p:TestAssemblies=false /p:TestPackages=true - - - ${{ if eq(parameters.isOfficialAllConfigurations, true) }}: - - librariesBuildArtifactName: 'libraries_bin_official_allconfigurations' - - ${{ if eq(parameters.isOfficialBuild, true) }}: - _msbuildCommonParameters: /p:OfficialBuildId=$(Build.BuildNumber) diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index 6ed1d936967663..04939ce42c5e29 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -8,29 +8,17 @@ parameters: crossrootfsDir: '' framework: 'net6.0' isOfficialBuild: false - isOfficialAllConfigurations: false runtimeVariant: '' platform: '' - - # When set to a non-empty value (Debug / Release), it determines the runtime's - # build configuration to use for building libraries and tests. Setting this - # property implies a dependency of this job on the appropriate runtime build - # and is used to construct the name of the Azure artifact representing - # runtime build to use for building the libraries and library tests. - liveRuntimeBuildConfig: '' - runtimeFlavor: 'coreclr' + testScope: '' timeoutInMinutes: 150 - preBuildSteps: [] container: '' condition: true dependOnEvaluatePaths: false shouldContinueOnError: false variables: {} pool: '' - runTests: false - useHelix: true - testScope: '' testBuildPlatforms: [] jobs: @@ -44,12 +32,9 @@ jobs: crossrootfsDir: ${{ parameters.crossrootfsDir }} framework: ${{ parameters.framework }} isOfficialBuild: ${{ parameters.isOfficialBuild }} - isOfficialAllConfigurations: ${{ parameters.isOfficialAllConfigurations }} - liveRuntimeBuildConfig: ${{ parameters.liveRuntimeBuildConfig }} runtimeFlavor: ${{ parameters.runtimeFlavor }} - runTests: ${{ parameters.runTests }} + runTests: false timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - preBuildSteps: ${{ parameters.preBuildSteps }} container: ${{ parameters.container }} condition: ${{ parameters.condition }} dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} @@ -59,33 +44,14 @@ jobs: name: build displayName: 'Build' - ${{ if and(ne(parameters.liveRuntimeBuildConfig, ''), eq(parameters.runTests, true)) }}: - dependsOn: - # Use full product dependency for test runs - - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} - variables: - librariesTestsArtifactName: ${{ format('libraries_test_assets_{0}_{1}_{2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }} - _subset: libs - _buildAction: '' - - _additionalBuildArguments: '' - - ${{ parameters.variables }} - - # Tests only run for 'allConfiguration' and 'net48' build-jobs - # If platform is in testBuildPlatforms we build tests as well. - - ${{ if or(eq(parameters.runTests, true), containsValue(parameters.testBuildPlatforms, parameters.platform)) }}: - - _subset: libs+libs.tests - - ${{ if eq(parameters.useHelix, false) }}: - - _buildAction: -restore -build -test - - ${{ if eq(parameters.useHelix, true) }}: - - _additionalBuildArguments: /p:ArchiveTests=true - + - _additionalBuildArguments: '/p:ArchiveTests=true' - ${{ parameters.variables }} steps: - - ${{ if eq(parameters.isOfficialBuild, true) }}: - - template: /eng/pipelines/common/restore-internal-tools.yml - - ${{ if in(parameters.osGroup, 'OSX', 'MacCatalyst', 'iOS', 'iOSSimulator', 'tvOS', 'tvOSSimulator') }}: - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh ${{ parameters.osGroup }} ${{ parameters.archType }} azDO displayName: Install Build Dependencies @@ -108,55 +74,26 @@ jobs: df -h displayName: Disk Usage after Build - - ${{ if eq(parameters.runTests, false) }}: - - template: /eng/pipelines/libraries/prepare-for-bin-publish.yml - parameters: - isOfficialBuild: ${{ parameters.isOfficialBuild }} - - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.ArtifactStagingDirectory)/artifacts - includeRootFolder: false - archiveType: $(archiveType) - archiveExtension: $(archiveExtension) - tarCompression: $(tarCompression) - artifactName: $(librariesBuildArtifactName) - displayName: Build Assets - - - ${{ if containsValue(parameters.testBuildPlatforms, parameters.platform) }}: - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.SourcesDirectory)/artifacts/helix - includeRootFolder: true - archiveType: $(archiveType) - archiveExtension: $(archiveExtension) - tarCompression: $(tarCompression) - artifactName: $(librariesTestsArtifactName) - displayName: Test Assets - - # Save AllConfigurations artifacts using the prepare-signed-artifacts format. The - # platform-specific jobs' nupkgs automatically flow through the matching platform-specific - # Installer build, but AllConfigurations should only be uploaded once, here. - - ${{ if eq(parameters.isOfficialAllConfigurations, true) }}: - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: Libraries_AllConfigurations - publishPackagesCondition: >- - or( - eq(variables['_librariesBuildProducedPackages'], true), - eq(variables['Build.SourceBranchName'], 'main'), - eq(variables['System.PullRequest.TargetBranch'], 'main')) - - - ${{ if and(eq(parameters.runTests, true), eq(parameters.useHelix, true)) }}: - - template: /eng/pipelines/libraries/helix.yml - parameters: - osGroup: ${{ parameters.osGroup }} - targetRid: ${{ parameters.targetRid }} - archType: ${{ parameters.archType }} - buildConfig: ${{ parameters.buildConfig }} - helixQueues: ${{ parameters.helixQueues }} - testScope: ${{ parameters.testScope }} - shouldContinueOnError: ${{ parameters.shouldContinueOnError }} - creator: dotnet-bot - testRunNamePrefixSuffix: $(_testRunNamePrefixSuffix) - extraHelixArguments: $(_extraHelixArguments) + - template: /eng/pipelines/libraries/prepare-for-bin-publish.yml + + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.ArtifactStagingDirectory)/artifacts + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: $(librariesBuildArtifactName) + displayName: Build Assets + + # Upload test assets + # We'll pull them down in another job to send to Helix + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/helix + includeRootFolder: true + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: $(librariesTestsArtifactName) + displayName: Test Assets \ No newline at end of file diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index ba8c49a70ffd2b..90235ef9921fc8 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -67,11 +67,7 @@ jobs: - ${{ if ne(parameters.dependsOn[0], '') }}: - ${{ parameters.dependsOn }} - ${{ if eq(parameters.dependsOn[0], '') }}: - - ${{ if notIn(parameters.framework, 'allConfigurations', 'net48') }}: - - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - # tests are built as part of product build - - ${{ if or(ne(parameters.archType, parameters.dependsOnTestArchitecture), ne(parameters.buildConfig, parameters.dependsOnTestBuildConfiguration), ne(parameters.osSubgroup, parameters.dependsOnTestOsSubgroup)) }}: - - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.dependsOnTestOsSubgroup, parameters.dependsOnTestArchitecture, parameters.dependsOnTestBuildConfiguration) }} + - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index f5267414df7080..3deb44c621ba7a 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -432,9 +432,10 @@ stages: buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -pgoinstrument isOfficialBuild: ${{ variables.isOfficialBuild }} nameSuffix: PGO - extraStepsTemplate: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - extraStepsParameters: - name: PGO + postBuildSteps: + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: PGO timeoutInMinutes: 95 # diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 880c143dd7ef58..b2ecde25bde1f7 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -806,9 +806,12 @@ jobs: eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), eq(variables['isFullMatrix'], true)) +# +# Build and test libraries for .NET Framework +# - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: Release platforms: - windows_x86 @@ -818,24 +821,33 @@ jobs: jobParameters: isFullMatrix: ${{ variables.isFullMatrix }} framework: net48 - runTests: true - testScope: innerloop + buildArgs: -s tools+libs+libs.tests -framework net48 -c $(_BuildConfig) -testscope innerloop /p:ArchiveTests=true + nameSuffix: Libraries_NET48 + timeoutInMinutes: 150 + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: NET48_$(_BuildConfig) + extraHelixArguments: /p:BuildTargetFramework=net48 condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), eq(variables['isFullMatrix'], true)) +# +# Build and test libraries AllConfigurations +# - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} platforms: - windows_x64 jobParameters: - isFullMatrix: ${{ variables.isFullMatrix }} - framework: allConfigurations - runTests: true - useHelix: false + buildArgs: -test -s tools+libs+libs.tests -allConfigurations -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true + nameSuffix: Libraries_AllConfigurations + timeoutInMinutes: 150 condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), From 0d3a86a045d43ffba022cc09cddde1a582c1a813 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Tue, 4 Jun 2024 16:47:29 -0400 Subject: [PATCH 2/3] Remove tools subset --- eng/pipelines/runtime.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index b2ecde25bde1f7..bfa1044e64bc75 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -821,7 +821,7 @@ jobs: jobParameters: isFullMatrix: ${{ variables.isFullMatrix }} framework: net48 - buildArgs: -s tools+libs+libs.tests -framework net48 -c $(_BuildConfig) -testscope innerloop /p:ArchiveTests=true + buildArgs: -s libs+libs.tests -framework net48 -c $(_BuildConfig) -testscope innerloop /p:ArchiveTests=true nameSuffix: Libraries_NET48 timeoutInMinutes: 150 postBuildSteps: @@ -845,7 +845,7 @@ jobs: platforms: - windows_x64 jobParameters: - buildArgs: -test -s tools+libs+libs.tests -allConfigurations -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true + buildArgs: -test -s libs+libs.tests -allConfigurations -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true nameSuffix: Libraries_AllConfigurations timeoutInMinutes: 150 condition: >- From bd30dedbb3a0b70698c901bf7651fb08f4358b84 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Tue, 4 Jun 2024 20:00:28 -0400 Subject: [PATCH 3/3] Build libs.tests --- eng/pipelines/libraries/build-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index 04939ce42c5e29..7577004470f945 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -46,7 +46,7 @@ jobs: variables: - librariesTestsArtifactName: ${{ format('libraries_test_assets_{0}_{1}_{2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }} - - _subset: libs + - _subset: libs+libs.tests - _buildAction: '' - _additionalBuildArguments: '/p:ArchiveTests=true' - ${{ parameters.variables }}