From 7f2b90352457792aad49835835f30a79fbd2b5ff Mon Sep 17 00:00:00 2001 From: Tomas Date: Thu, 28 Nov 2019 17:14:30 +0100 Subject: [PATCH 01/25] Live-live pipeline rebased against Santiago's CoreFX build change --- eng/pipelines/coreclr/pr.yml | 159 ++---------------- .../coreclr/templates/build-test-job.yml | 21 ++- .../coreclr/templates/run-test-job.yml | 19 ++- .../coreclr/templates/xplat-pipeline-job.yml | 13 ++ eng/pipelines/libraries/base-job.yml | 5 +- eng/pipelines/libraries/build-job.yml | 29 ++++ src/coreclr/build-test.cmd | 8 +- src/coreclr/build-test.sh | 23 ++- src/coreclr/tests/scripts/patch-corefx.py | 4 +- 9 files changed, 115 insertions(+), 166 deletions(-) mode change 100644 => 100755 src/coreclr/tests/scripts/patch-corefx.py diff --git a/eng/pipelines/coreclr/pr.yml b/eng/pipelines/coreclr/pr.yml index 360b58dcb52bbd..3dee2a10277f2f 100644 --- a/eng/pipelines/coreclr/pr.yml +++ b/eng/pipelines/coreclr/pr.yml @@ -4,24 +4,6 @@ pr: branches: include: - master - - release/*.* - paths: - include: - - '*' - - src/libraries/System.Private.CoreLib/* - exclude: - - docs/* - - CODE-OF-CONDUCT.md - - CONTRIBUTING.md - - LICENSE.TXT - - PATENTS.TXT - - README.md - - SECURITY.md - - THIRD-PARTY-NOTICES.TXT - - src/installer/* - - src/libraries/* - - eng/pipelines/installer/* - - eng/pipelines/libraries/* jobs: # @@ -30,176 +12,63 @@ jobs: - template: /eng/pipelines/common/checkout-job.yml # -# Debug builds -# -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml - buildConfig: debug - platforms: - - Windows_NT_x64 - - Windows_NT_x86 - jobParameters: - testGroup: innerloop - -# -# Checked builds +# Build CoreCLR # - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml buildConfig: checked platforms: - - Linux_arm - - Linux_arm64 - - Linux_musl_x64 - - Linux_x64 - OSX_x64 - - Windows_NT_arm - - Windows_NT_arm64 - - Windows_NT_x64 - Windows_NT_x86 - jobParameters: - testGroup: innerloop - -# -# Release builds -# -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml - buildConfig: release - platforms: - - Linux_arm64 - - Linux_musl_x64 - - Linux_rhel6_x64 - - OSX_x64 - - Windows_NT_arm - Windows_NT_arm64 - - Windows_NT_x64 jobParameters: testGroup: innerloop # -# Checked test builds +# Build CoreFX using live CoreLib from CoreCLR # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-test-job.yml + jobTemplate: /eng/pipelines/libraries/build-job.yml buildConfig: checked platforms: - - Linux_arm - - Linux_arm64 - OSX_x64 - - Windows_NT_arm - - Windows_NT_arm64 - - Windows_NT_x64 - Windows_NT_x86 - jobParameters: - testGroup: innerloop - -# -# Checked JIT test executions -# -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/run-test-job.yml - buildConfig: checked - platforms: - - Linux_arm - - Linux_arm64 - - Linux_musl_x64 - - Linux_x64 - - OSX_x64 - - Windows_NT_arm - Windows_NT_arm64 - - Windows_NT_x64 - - Windows_NT_x86 - helixQueueGroup: pr - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: - testGroup: innerloop + isOfficialBuild: false + testScope: empty + useLiveCoreClr: true # -# Checked R2R test executions +# Test builds using live CoreFX # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/run-test-job.yml + jobTemplate: /eng/pipelines/coreclr/templates/build-test-job.yml buildConfig: checked platforms: - - Linux_x64 - OSX_x64 - - Windows_NT_x64 - Windows_NT_x86 - helixQueueGroup: pr - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + - Windows_NT_arm64 jobParameters: testGroup: innerloop - readyToRun: true - displayNameArgs: R2R + useLiveLibraries: true # -# CoreFX test runs against CoreCLR +# CoreCLR Test executions # - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/coreclr/templates/run-test-job.yml buildConfig: checked platforms: - - Linux_x64 - - Windows_NT_x64 - helixQueueGroup: pr - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: innerloop - corefxTests: true - displayNameArgs: CoreFX - -# -# Crossgen-comparison jobs -# -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/crossgen-comparison-job.yml - buildConfig: checked - platforms: - - Linux_arm - helixQueueGroup: pr - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - -# -# Release test builds -# -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-test-job.yml - buildConfig: release - platforms: - OSX_x64 - jobParameters: - testGroup: innerloop - -# -# Release test builds -# -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/run-test-job.yml - buildConfig: release - platforms: - - Linux_musl_x64 + - Windows_NT_x86 + - Windows_NT_arm64 helixQueueGroup: pr helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: testGroup: innerloop - -# -# Formatting -# -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/format-job.yml - platforms: - - Linux_x64 - - Windows_NT_x64 + useLiveLibraries: true diff --git a/eng/pipelines/coreclr/templates/build-test-job.yml b/eng/pipelines/coreclr/templates/build-test-job.yml index bb30b83e8c18b4..d9175e7b66c379 100644 --- a/eng/pipelines/coreclr/templates/build-test-job.yml +++ b/eng/pipelines/coreclr/templates/build-test-job.yml @@ -5,6 +5,7 @@ parameters: osSubgroup: '' container: '' testGroup: '' + useLiveLibraries: false displayNameArgs: '' condition: true stagedBuild: false @@ -31,6 +32,7 @@ jobs: container: ${{ parameters.container }} testGroup: ${{ parameters.testGroup }} stagedBuild: ${{ parameters.stagedBuild }} + useLiveLibraries: ${{ parameters.useLiveLibraries }} variables: ${{ parameters.variables }} pool: ${{ parameters.pool }} @@ -55,7 +57,11 @@ jobs: # because it needs System.Private.Corelib; we should be able to remove this dependency # by switching over to using reference assembly. ${{ if ne(parameters.stagedBuild, true) }}: - dependsOn: ${{ format('coreclr_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + dependsOn: + - ${{ format('coreclr_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ if eq(parameters.useLiveLibraries, true) }}: + - ${{ format('libraries_build_{0}_{1}{2}_{3}_{4}', 'netcoreapp', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + ${{ if eq(parameters.testGroup, 'innerloop') }}: timeoutInMinutes: 90 @@ -75,6 +81,17 @@ jobs: displayName: Install native dependencies + # Optionally download live-built CoreFX artifacts + - ${{ if eq(parameters.useLiveLibraries, true) }}: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(coreFxDownloadDir) + cleanUnpackFolder: false + artifactFileName: '$(coreFxArtifactName)$(archiveExtension)' + artifactName: '$(coreFxArtifactName)' + displayName: 'live-built CoreFX' + + # Download product binaries directory - template: /eng/pipelines/common/download-artifact-step.yml parameters: @@ -85,7 +102,7 @@ jobs: # Build managed test components - - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipnative skipgeneratelayout $(buildConfig) $(archType) $(crossArg) $(priorityArg) ci + - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipnative skipgeneratelayout $(buildConfig) $(archType) $(crossArg) $(priorityArg) ci $(coreFxOverrideArg) displayName: Build managed test components diff --git a/eng/pipelines/coreclr/templates/run-test-job.yml b/eng/pipelines/coreclr/templates/run-test-job.yml index f65e2fa9da75da..461bef23a3f997 100644 --- a/eng/pipelines/coreclr/templates/run-test-job.yml +++ b/eng/pipelines/coreclr/templates/run-test-job.yml @@ -9,6 +9,7 @@ parameters: testGroup: '' crossrootfsDir: '' readyToRun: false + useLiveLibraries: false crossgen2: false helixQueues: '' # If true, run the corefx tests instead of the coreclr ones @@ -38,6 +39,7 @@ jobs: crossrootfsDir: ${{ parameters.crossrootfsDir }} corefxTests: ${{ parameters.corefxTests }} stagedBuild: ${{ parameters.stagedBuild }} + useLiveLibraries: ${{ parameters.useLiveLibraries }} helixType: 'build/tests/' pool: ${{ parameters.pool }} @@ -105,6 +107,17 @@ jobs: steps: + # Optionally download live-built CoreFX artifacts + - ${{ if eq(parameters.useLiveLibraries, true) }}: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(coreFxDownloadDir) + cleanUnpackFolder: false + artifactFileName: '$(coreFxArtifactName)$(archiveExtension)' + artifactName: '$(coreFxArtifactName)' + displayName: 'live-built CoreFX' + + # Download and unzip managed test artifacts - ${{ if ne(parameters.corefxTests, true) }}: - template: /eng/pipelines/common/download-artifact-step.yml @@ -151,19 +164,19 @@ jobs: # the native artifacts to the final test folders is dependent on availability of the # managed test artifacts. - ${{ if ne(parameters.corefxTests, true) }}: - - script: $(coreClrRepoRootDir)build-test$(scriptExt) copynativeonly $(buildConfig) $(archType) $(priorityArg) + - script: $(coreClrRepoRootDir)build-test$(scriptExt) copynativeonly $(buildConfig) $(archType) $(priorityArg) $(coreFxOverrideArg) displayName: Copy native test components to test output folder # Generate test host - ${{ if eq(parameters.corefxTests, true) }}: - - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipmanaged skipnative $(testhostArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) + - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipmanaged skipnative $(testhostArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(coreFxOverrideArg) displayName: Generate test host # Crossgen framework assemblies prior to triggering readyToRun execution runs. - ${{ if eq(parameters.readyToRun, true) }}: - - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipmanaged skipnative $(crossgenArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) + - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipmanaged skipnative $(crossgenArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(coreFxOverrideArg) displayName: Crossgen framework assemblies diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index f642b79db0d645..de2cda2199f32a 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -11,6 +11,7 @@ parameters: testGroup: '' crossrootfsDir: '' corefxTests: false + useLiveLibraries: false stagedBuild: false strategy: '' pool: '' @@ -103,6 +104,18 @@ jobs: - name: priorityArg value: '-priority=1' + - coreFxArtifactName: '' + - coreFxOverrideArg: '' + - coreFxDownloadDir: '' + + - ${{ if eq(parameters.useLiveLibraries, true) }}: + - coreFxArtifactName: ${{ format('libraries_bin_{0}_{1}{2}_{3}_{4}', 'netcoreapp', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - coreFxDownloadDir: $(Build.SourcesDirectory)/artifacts + - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: + - coreFxOverrideArg: ' localcorefxpath $(Build.SourcesDirectory)' + - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: + - coreFxOverrideArg: ' localcorefxpath=$(Build.SourcesDirectory)' + - ${{ each variable in parameters.variables }}: - ${{insert}}: ${{ variable }} diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index e345b8973f81ff..56671cccf3e0d7 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -30,7 +30,10 @@ jobs: helixRepo: dotnet/runtime pool: ${{ parameters.pool }} variables: - - _BuildConfig: ${{ parameters.buildConfig }} + - _BuildConfig: Debug + - ${{ if eq(parameters.buildConfig, 'release') }}: + - _BuildConfig: Release + - _msbuildCommonParameters: '' - _stripSymbolsArg: '' - _runtimeOSArg: '' diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index 0ae7cc6093d34b..50d8e038e1f901 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -5,6 +5,7 @@ parameters: archType: '' framework: netcoreapp isOfficialBuild: false + useLiveCoreClr: false timeoutInMinutes: 150 preBuildSteps: [] container: '' @@ -29,6 +30,10 @@ jobs: testScope: ${{ parameters.testScope }} stepName: Build + ${{ if eq(parameters.useLiveCoreClr, true) }}: + dependsOn: + - ${{ format('coreclr_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + variables: - _skipTestRestoreArg: /p:SkipTestRestore=true - _buildAction: -build @@ -40,7 +45,31 @@ jobs: - ${{ if ne(parameters.framework, 'allConfigurations') }}: - _skipTestRestoreArg: /p:SkipTestRestore=false + - _coreClrArtifactName: '' + - _coreClrDownloadPath: '' + - _coreClrOverridePathArg: '' + + - ${{ if eq(parameters.useLiveCoreClr, true) }}: + - _coreClrDownloadPath: '$(Build.SourcesDirectory)/artifacts/transport/coreclr' + - _coreClrArtifactName: 'CoreCLRProduct_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}' + - _coreClrOverridePathArg: ' /p:CoreCLROverridePath=$(_coreClrDownloadPath)' + - _skipTestRestoreArg: /p:SkipTestRestore=true $(_coreClrOverridePathArg) + + - _archiveExtension: '.tar.gz' + - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: + - _archiveExtension: '.zip' + + - ${{ parameters.variables }} + steps: + - ${{ if eq(parameters.useLiveCoreClr, true) }}: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(_coreClrDownloadPath) + artifactFileName: '$(_coreClrArtifactName)$(_archiveExtension)' + artifactName: '$(_coreClrArtifactName)' + displayName: 'CoreCLR build drop' + - script: $(_buildScript) -restore $(_buildArguments) $(_skipTestRestoreArg) displayName: Restore diff --git a/src/coreclr/build-test.cmd b/src/coreclr/build-test.cmd index f126181d88b681..6a49fddce48981 100644 --- a/src/coreclr/build-test.cmd +++ b/src/coreclr/build-test.cmd @@ -481,6 +481,8 @@ REM === The test host includes a dotnet executable, system libraries and CoreCLR REM === REM ========================================================================================= +if not "%__LocalCoreFXPath%" == "" goto SkipBuildingCoreFXTestHost + echo %__MsgPrefix%Building CoreFX test host set __BuildLogRootName=Tests_CoreFX_Testhost @@ -505,6 +507,8 @@ if errorlevel 1 ( exit /b 1 ) +:SkipBuildingCoreFXTestHost + REM ========================================================================================= REM === REM === Create test wrappers. @@ -577,8 +581,10 @@ REM ============================================================================ if NOT "%__LocalCoreFXPath%"=="" ( echo Patch CoreFX from %__LocalCoreFXPath% + set __CoreFXBuildType=Debug + if "!__BuildType!" == "release" (set __CoreFXBuildType=Release) set NEXTCMD=python "%__ProjectDir%\tests\scripts\patch-corefx.py" -clr_core_root "%CORE_ROOT%"^ - -fx_root "%__LocalCoreFXPath%" -arch %__BuildArch% -build_type %__BuildType% + -fx_root "%__LocalCoreFXPath%" -arch %__BuildArch% -build_type %__CoreFXBuildType% echo !NEXTCMD! !NEXTCMD! ) diff --git a/src/coreclr/build-test.sh b/src/coreclr/build-test.sh index 3f0fc0634b5150..f0abe91289a32f 100755 --- a/src/coreclr/build-test.sh +++ b/src/coreclr/build-test.sh @@ -136,7 +136,12 @@ patch_corefx_libraries() { echo "${__MsgPrefix}Patching CORE_ROOT: '${CORE_ROOT}' with CoreFX libaries from enlistment '${__LocalCoreFXPath}" - patchCoreFXArguments=("-clr_core_root" "${CORE_ROOT}" "-fx_root" "${__LocalCoreFXPath}" "-arch" "${__BuildArch}" "-build_type" "${__BuildType}") + __CoreFXBuildType=Debug + if [ "$__BuildType" == "release" ]; then + __CoreFXBuildType=Release + fi + + patchCoreFXArguments=("-clr_core_root" "${CORE_ROOT}" "-fx_root" "${__LocalCoreFXPath}" "-arch" "${__BuildArch}" "-build_type" "${__CoreFXBuildType}") scriptPath="$__ProjectDir/tests/scripts" echo "python ${scriptPath}/patch-corefx.py ${patchCoreFXArguments[@]}" $__Python "${scriptPath}/patch-corefx.py" "${patchCoreFXArguments[@]}" @@ -371,10 +376,10 @@ build_Tests() if [ $__SkipGenerateLayout != 1 ]; then generate_layout - fi - if [ ! -z "$__LocalCoreFXPath" ]; then - patch_corefx_libraries + if [ ! -z "$__LocalCoreFXPath" ]; then + patch_corefx_libraries + fi fi } @@ -630,14 +635,8 @@ handle_arguments() { __SkipGenerateLayout=1 ;; - localcorefxpath) - if [ -n "$2" ]; then - __LocalCoreFXPath="$2" - shift - else - echo "ERROR: 'localcorefxpath' requires a non-empty option argument" - exit 1 - fi + localcorefxpath=*|-localcorefxpath=*) + __LocalCoreFXPath=$(echo "$1" | cut -d'=' -f 2) ;; *) diff --git a/src/coreclr/tests/scripts/patch-corefx.py b/src/coreclr/tests/scripts/patch-corefx.py old mode 100644 new mode 100755 index bbc44c1af07737..a5e42b6ae1e5b3 --- a/src/coreclr/tests/scripts/patch-corefx.py +++ b/src/coreclr/tests/scripts/patch-corefx.py @@ -239,7 +239,7 @@ def main(args): clr_os = 'Windows_NT' if Is_windows else Unix_name_map[os.uname()[0]] if not os.path.exists(clr_core_root): - raise Exception('Core root path %s does not exist.' % (core_root)) + raise Exception('Core root path %s does not exist.' % (clr_core_root)) fx_bin = os.path.join(fx_root, 'artifacts', @@ -248,7 +248,7 @@ def main(args): 'netcoreapp-%s-%s-%s' % (clr_os, build_type, arch)) if not os.path.exists(fx_bin): - raise Exception('CoreFX bin path %s does not exist.' % (core_root)) + raise Exception('CoreFX bin path %s does not exist.' % (fx_bin)) patch_coreclr_root(clr_core_root, fx_bin) From 275ee151ea8a7a9fff6a33ae7b669d68aef8e094 Mon Sep 17 00:00:00 2001 From: Tomas Date: Thu, 28 Nov 2019 19:28:33 +0100 Subject: [PATCH 02/25] Fix library script name --- eng/pipelines/libraries/base-job.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index 56671cccf3e0d7..a0afe5a84bf1f2 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -30,6 +30,7 @@ jobs: helixRepo: dotnet/runtime pool: ${{ parameters.pool }} variables: + - buildScriptFileName: libraries - _BuildConfig: Debug - ${{ if eq(parameters.buildConfig, 'release') }}: - _BuildConfig: Release From 8336382f1d30ffa75035ea8316f7adca9193f36f Mon Sep 17 00:00:00 2001 From: Tomas Date: Thu, 28 Nov 2019 19:52:44 +0100 Subject: [PATCH 03/25] One more configuration mapping fix for CoreFX build --- eng/pipelines/libraries/base-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index a0afe5a84bf1f2..8ea33d7fe5f7ef 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -81,7 +81,7 @@ jobs: - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - _stripSymbolsArg: -stripSymbols - - _buildArguments: -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) $(_stripSymbolsArg) $(_testScopeArg) $(_warnAsErrorArg) $(_runtimeOSArg) $(_portableArg) $(_msbuildCommonParameters) + - _buildArguments: -configuration $(_BuildConfig) -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) $(_stripSymbolsArg) $(_testScopeArg) $(_warnAsErrorArg) $(_runtimeOSArg) $(_portableArg) $(_msbuildCommonParameters) - ${{ parameters.variables }} dependsOn: From 1626467d8817f928da7d01f0b27e1181d971cf32 Mon Sep 17 00:00:00 2001 From: Tomas Date: Thu, 28 Nov 2019 23:20:42 +0100 Subject: [PATCH 04/25] Fix Windows build-test script, expand build matrix --- eng/pipelines/coreclr/pr.yml | 8 ++++++++ src/coreclr/build-test.cmd | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/pr.yml b/eng/pipelines/coreclr/pr.yml index 3dee2a10277f2f..9e6791d4a911e0 100644 --- a/eng/pipelines/coreclr/pr.yml +++ b/eng/pipelines/coreclr/pr.yml @@ -20,7 +20,9 @@ jobs: buildConfig: checked platforms: - OSX_x64 + - Linux_x64 - Windows_NT_x86 + - Windows_NT_x64 - Windows_NT_arm64 jobParameters: testGroup: innerloop @@ -34,7 +36,9 @@ jobs: buildConfig: checked platforms: - OSX_x64 + - Linux_x64 - Windows_NT_x86 + - Windows_NT_x64 - Windows_NT_arm64 jobParameters: isOfficialBuild: false @@ -50,7 +54,9 @@ jobs: buildConfig: checked platforms: - OSX_x64 + - Linux_x64 - Windows_NT_x86 + - Windows_NT_x64 - Windows_NT_arm64 jobParameters: testGroup: innerloop @@ -65,7 +71,9 @@ jobs: buildConfig: checked platforms: - OSX_x64 + - Linux_x64 - Windows_NT_x86 + - Windows_NT_x64 - Windows_NT_arm64 helixQueueGroup: pr helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml diff --git a/src/coreclr/build-test.cmd b/src/coreclr/build-test.cmd index 6a49fddce48981..dfec36661d5c35 100644 --- a/src/coreclr/build-test.cmd +++ b/src/coreclr/build-test.cmd @@ -582,9 +582,9 @@ REM ============================================================================ if NOT "%__LocalCoreFXPath%"=="" ( echo Patch CoreFX from %__LocalCoreFXPath% set __CoreFXBuildType=Debug - if "!__BuildType!" == "release" (set __CoreFXBuildType=Release) + if "%__BuildType%" == "release" (set __CoreFXBuildType=Release) set NEXTCMD=python "%__ProjectDir%\tests\scripts\patch-corefx.py" -clr_core_root "%CORE_ROOT%"^ - -fx_root "%__LocalCoreFXPath%" -arch %__BuildArch% -build_type %__CoreFXBuildType% + -fx_root "%__LocalCoreFXPath%" -arch %__BuildArch% -build_type !__CoreFXBuildType! echo !NEXTCMD! !NEXTCMD! ) From 92478273a7af28689ef3ee48a8c2832a3d578336 Mon Sep 17 00:00:00 2001 From: Tomas Date: Fri, 29 Nov 2019 17:10:52 +0100 Subject: [PATCH 05/25] Add jobs for CoreFX test execution, experimentally downgrade Python --- eng/pipelines/coreclr/pr.yml | 39 ++++++++++++++++++++++ eng/pipelines/libraries/base-job.yml | 19 ++++++++--- eng/pipelines/libraries/build-job.yml | 22 +++--------- eng/pipelines/libraries/build-test-job.yml | 9 +++-- eng/pipelines/libraries/run-test-job.yml | 11 +++--- src/coreclr/tests/scripts/patch-corefx.py | 2 +- 6 files changed, 73 insertions(+), 29 deletions(-) diff --git a/eng/pipelines/coreclr/pr.yml b/eng/pipelines/coreclr/pr.yml index 9e6791d4a911e0..860f3978a9bd6f 100644 --- a/eng/pipelines/coreclr/pr.yml +++ b/eng/pipelines/coreclr/pr.yml @@ -80,3 +80,42 @@ jobs: jobParameters: testGroup: innerloop useLiveLibraries: true + +# +# CoreFX Test Build +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build-test-job.yml + buildConfig: checked + platforms: + - OSX_x64 + - Linux_x64 + - Windows_NT_x86 + - Windows_NT_x64 + - Windows_NT_arm64 + jobParameters: + isOfficialBuild: false + testScope: innerloop + useLiveCoreClr: true + +# +# CoreFX Test Execution +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/run-test-job.yml + buildConfig: checked + platforms: + - OSX_x64 + - Linux_x64 + - Windows_NT_x86 + - Windows_NT_x64 + - Windows_NT_arm64 + helixQueueGroup: pr + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + jobParameters: + isOfficialBuild: false + testScope: innerloop + useLiveCoreClr: true + dependsOnBuildConfiguration: checked diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index 8ea33d7fe5f7ef..aa946e617575e2 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -5,6 +5,7 @@ parameters: osSubgroup: '' framework: '' isOfficialBuild: false + useLiveCoreClr: false timeoutInMinutes: 150 container: '' steps: [] @@ -14,16 +15,15 @@ parameters: displayName: '' testScope: '' pool: '' - stepName: '' jobs: - template: /eng/common/templates/job/job.yml parameters: - name: ${{ format('libraries_{0}_{1}_{2}{3}_{4}_{5}', parameters.stepName, parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + name: ${{ format('libraries_{0}_{1}_{2}{3}_{4}_{5}', parameters.name, parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} ${{ if eq(parameters.framework, 'netcoreapp') }}: - displayName: ${{ format('{0} {1}{2} {3} {4}', parameters.stepName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('Libraries {0} {1}{2} {3} {4}', parameters.displayName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} ${{ if ne(parameters.framework, 'netcoreapp') }}: - displayName: ${{ format('{0} {1} {2} {3} {4}', parameters.stepName, parameters.osGroup, parameters.framework, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('Libraries {0} {1} {2} {3} {4}', parameters.displayName, parameters.osGroup, parameters.framework, parameters.archType, parameters.buildConfig) }} enableTelemetry: ${{ parameters.isOfficialBuild }} # TODO: figure out if it's needed container: ${{ parameters.container }} @@ -69,15 +69,26 @@ jobs: - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - _msbuildCommonParameters: /p:OfficialBuildId=$(Build.BuildNumber) + - _coreClrArtifactName: '' + - _coreClrDownloadPath: '' + - _coreClrOverridePathArg: '' + + - ${{ if eq(parameters.useLiveCoreClr, true) }}: + - _coreClrDownloadPath: '$(Build.SourcesDirectory)/artifacts/transport/coreclr' + - _coreClrArtifactName: 'CoreCLRProduct_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}' + - _coreClrOverridePathArg: ' /p:CoreCLROverridePath=$(_coreClrDownloadPath)' + # Windows variables - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - _msbuildCommand: powershell -ExecutionPolicy ByPass -NoProfile eng\common\msbuild.ps1 -warnaserror:0 -ci - _runtimeOSArg: /p:RuntimeOS=win10 + - _archiveExtension: '.zip' # Non-Windows variables - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - _msbuildCommand: ./eng/common/msbuild.sh --warnaserror false --ci - _buildScript: ./$(buildScriptFileName)$(scriptExt) + - _archiveExtension: '.tar.gz' - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - _stripSymbolsArg: -stripSymbols diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index 50d8e038e1f901..21add5548d9712 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -23,12 +23,14 @@ jobs: archType: ${{ parameters.archType }} framework: ${{ parameters.framework }} isOfficialBuild: ${{ parameters.isOfficialBuild }} + useLiveCoreClr: ${{ parameters.useLiveCoreClr }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} preBuildSteps: ${{ parameters.preBuildSteps }} container: ${{ parameters.container }} pool: ${{ parameters.pool }} testScope: ${{ parameters.testScope }} - stepName: Build + name: build + displayName: 'Build' ${{ if eq(parameters.useLiveCoreClr, true) }}: dependsOn: @@ -42,23 +44,9 @@ jobs: - EMSDK_PATH: $(Build.BinariesDirectory)/emsdk - ${{ if eq(parameters.runTests, true) }}: - _buildAction: -build -buildtests /p:ArchiveTests=true - - ${{ if ne(parameters.framework, 'allConfigurations') }}: + - ${{ if and(ne(parameters.framework, 'allConfigurations'), not(parameters.useLiveCoreClr)) }}: - _skipTestRestoreArg: /p:SkipTestRestore=false - - _coreClrArtifactName: '' - - _coreClrDownloadPath: '' - - _coreClrOverridePathArg: '' - - - ${{ if eq(parameters.useLiveCoreClr, true) }}: - - _coreClrDownloadPath: '$(Build.SourcesDirectory)/artifacts/transport/coreclr' - - _coreClrArtifactName: 'CoreCLRProduct_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}' - - _coreClrOverridePathArg: ' /p:CoreCLROverridePath=$(_coreClrDownloadPath)' - - _skipTestRestoreArg: /p:SkipTestRestore=true $(_coreClrOverridePathArg) - - - _archiveExtension: '.tar.gz' - - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - _archiveExtension: '.zip' - - ${{ parameters.variables }} steps: @@ -70,7 +58,7 @@ jobs: artifactName: '$(_coreClrArtifactName)' displayName: 'CoreCLR build drop' - - script: $(_buildScript) -restore $(_buildArguments) $(_skipTestRestoreArg) + - script: $(_buildScript) -restore $(_buildArguments) $(_skipTestRestoreArg) $(_coreClrOverridePathArg) displayName: Restore - ${{ if eq(parameters.isOfficialBuild, true) }}: diff --git a/eng/pipelines/libraries/build-test-job.yml b/eng/pipelines/libraries/build-test-job.yml index a847b40cf7e235..bedc5714d121f8 100644 --- a/eng/pipelines/libraries/build-test-job.yml +++ b/eng/pipelines/libraries/build-test-job.yml @@ -5,6 +5,7 @@ parameters: archType: '' framework: netcoreapp isOfficialBuild: false + useLiveCoreClr: false timeoutInMinutes: 150 container: '' publishTestArtifacs: true @@ -22,14 +23,16 @@ jobs: archType: ${{ parameters.archType }} framework: ${{ parameters.framework }} isOfficialBuild: ${{ parameters.isOfficialBuild }} + useLiveCoreClr: ${{ parameters.useLiveCoreClr }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} container: ${{ parameters.container }} pool: ${{ parameters.pool }} testScope: ${{ parameters.testScope }} - stepName: Test + name: test_build + displayName: 'Test Build' dependsOn: - - ${{ format('libraries_Build_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ format('libraries_build_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} variables: - _archiveTestsParameter: /p:ArchiveTests=true @@ -45,7 +48,7 @@ jobs: artifactFileName: ${{ format('libraries_bin_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}$(archiveExtension) unpackFolder: $(Build.SourcesDirectory)/artifacts - - script: $(_buildScript) -restore $(_buildArguments) $(_skipTestRestoreArg) + - script: $(_buildScript) -restore $(_buildArguments) $(_skipTestRestoreArg) $(_coreClrOverridePathArg) displayName: Restore - script: $(_buildScript) diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index d7b316af4e9555..ffb0accf56d663 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -5,6 +5,7 @@ parameters: archType: '' framework: netcoreapp isOfficialBuild: false + useLiveCoreClr: false timeoutInMinutes: 150 pool: '' testScope: '' @@ -21,14 +22,16 @@ jobs: archType: ${{ parameters.archType }} framework: ${{ parameters.framework }} isOfficialBuild: ${{ parameters.isOfficialBuild }} + useLiveCoreClr: ${{ parameters.useLiveCoreClr }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} container: '' # we just send to helix, no need to use a container. pool: ${{ parameters.pool }} - stepName: Run + name: test_run + displayName: 'Test Run' dependsOn: - - ${{ format('libraries_Build_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ format('libraries_Test_{0}_{1}_x64_{2}', parameters.framework, parameters.osGroup, parameters.dependsOnBuildConfiguration) }} + - ${{ format('libraries_build_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ format('libraries_test_build_{0}_{1}_x64_{2}', parameters.framework, parameters.osGroup, parameters.dependsOnBuildConfiguration) }} variables: - _archiveTestsParameter: /p:ArchiveTests=true @@ -61,4 +64,4 @@ jobs: framework: ${{ parameters.framework }} testScope: ${{ parameters.testScope }} creator: dotnet-bot - helixToken: '' \ No newline at end of file + helixToken: '' diff --git a/src/coreclr/tests/scripts/patch-corefx.py b/src/coreclr/tests/scripts/patch-corefx.py index a5e42b6ae1e5b3..54dd929892e4c1 100755 --- a/src/coreclr/tests/scripts/patch-corefx.py +++ b/src/coreclr/tests/scripts/patch-corefx.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # # Licensed to the .NET Foundation under one or more agreements. # The .NET Foundation licenses this file to you under the MIT license. From ac47a03406d161b2b54f3ba398c5e247b9d28892 Mon Sep 17 00:00:00 2001 From: Tomas Date: Fri, 29 Nov 2019 18:21:11 +0100 Subject: [PATCH 06/25] Move download of live CoreCLR artifacts to base-job --- eng/pipelines/libraries/base-job.yml | 8 ++++++++ eng/pipelines/libraries/build-job.yml | 8 -------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index aa946e617575e2..4502cc36d17c68 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -117,4 +117,12 @@ jobs: git remote set-url origin $(Build.Repository.Uri) displayName: Clone the repository from Checkout.bundle + - ${{ if eq(parameters.useLiveCoreClr, true) }}: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(_coreClrDownloadPath) + artifactFileName: '$(_coreClrArtifactName)$(_archiveExtension)' + artifactName: '$(_coreClrArtifactName)' + displayName: 'CoreCLR build drop' + - ${{ parameters.steps }} diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index 21add5548d9712..01f343642cf240 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -50,14 +50,6 @@ jobs: - ${{ parameters.variables }} steps: - - ${{ if eq(parameters.useLiveCoreClr, true) }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(_coreClrDownloadPath) - artifactFileName: '$(_coreClrArtifactName)$(_archiveExtension)' - artifactName: '$(_coreClrArtifactName)' - displayName: 'CoreCLR build drop' - - script: $(_buildScript) -restore $(_buildArguments) $(_skipTestRestoreArg) $(_coreClrOverridePathArg) displayName: Restore From d2c0c8230ab6d29bc7e6b8f8c3c363068f4e81c4 Mon Sep 17 00:00:00 2001 From: Tomas Date: Fri, 29 Nov 2019 21:25:20 +0100 Subject: [PATCH 07/25] Don't clean up unpack folder in build-test-job as that nukes CoreCLR bits --- eng/pipelines/libraries/build-test-job.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/libraries/build-test-job.yml b/eng/pipelines/libraries/build-test-job.yml index bedc5714d121f8..bfa312b54023ad 100644 --- a/eng/pipelines/libraries/build-test-job.yml +++ b/eng/pipelines/libraries/build-test-job.yml @@ -47,6 +47,7 @@ jobs: artifactName: ${{ format('libraries_bin_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} artifactFileName: ${{ format('libraries_bin_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}$(archiveExtension) unpackFolder: $(Build.SourcesDirectory)/artifacts + cleanUnpackFolder: false - script: $(_buildScript) -restore $(_buildArguments) $(_skipTestRestoreArg) $(_coreClrOverridePathArg) displayName: Restore From 36b9784cfe307beaed8ecf578cbb90004d158917 Mon Sep 17 00:00:00 2001 From: Tomas Date: Sat, 30 Nov 2019 17:39:36 +0100 Subject: [PATCH 08/25] Fix path to sendtohelix.proj script --- eng/pipelines/libraries/helix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/libraries/helix.yml b/eng/pipelines/libraries/helix.yml index 4f5f2d28af3c45..945b51bfe72635 100644 --- a/eng/pipelines/libraries/helix.yml +++ b/eng/pipelines/libraries/helix.yml @@ -11,7 +11,7 @@ parameters: steps: - script: ${{ parameters.msbuildScript }} - $(sourcesRoot)/sendtohelix.proj + $(Build.SourcesDirectory)/src/libraries/sendtohelix.proj /t:test /p:ArchGroup=${{ parameters.archGroup }} /p:ConfigurationGroup=${{ parameters.configuration }} From 1c04de4d5d88792b22bac0722631ce15fbbc3658 Mon Sep 17 00:00:00 2001 From: Tomas Date: Sun, 1 Dec 2019 00:06:26 +0100 Subject: [PATCH 09/25] Specify libraries run more precisely to hopefully get some queues --- eng/pipelines/coreclr/pr.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/pipelines/coreclr/pr.yml b/eng/pipelines/coreclr/pr.yml index 860f3978a9bd6f..4681be90536d67 100644 --- a/eng/pipelines/coreclr/pr.yml +++ b/eng/pipelines/coreclr/pr.yml @@ -116,6 +116,8 @@ jobs: helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml jobParameters: isOfficialBuild: false + isFullMatrix: false + framework: netcoreapp testScope: innerloop useLiveCoreClr: true dependsOnBuildConfiguration: checked From 8048966451c78f44618d111153c9609eb8106fe6 Mon Sep 17 00:00:00 2001 From: Tomas Date: Sun, 1 Dec 2019 21:00:35 +0100 Subject: [PATCH 10/25] Instrumentation for chasing down missing Helix artifact (test runtime) --- src/libraries/sendtohelix.proj | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libraries/sendtohelix.proj b/src/libraries/sendtohelix.proj index ed0514b2de74b4..bcab75201c71de 100644 --- a/src/libraries/sendtohelix.proj +++ b/src/libraries/sendtohelix.proj @@ -84,6 +84,9 @@ Outputs="$(TestArchiveRuntimeFile)" Condition="'$(TargetGroup)' != 'AllConfigurations'"> + + + + + + + + From c8820f2817b5117a8c06619225ba2d7396792fbf Mon Sep 17 00:00:00 2001 From: Tomas Date: Mon, 2 Dec 2019 11:59:25 +0100 Subject: [PATCH 11/25] Experimentally import Directory.Build.props from sendtohelix.proj ... to see whether it fixes the missing property definitions for TestArchiveRuntimeRoot and others needed to compress the runtime directory as part of Helix publishing. --- src/libraries/sendtohelix.proj | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libraries/sendtohelix.proj b/src/libraries/sendtohelix.proj index bcab75201c71de..426386341ffd11 100644 --- a/src/libraries/sendtohelix.proj +++ b/src/libraries/sendtohelix.proj @@ -1,4 +1,7 @@ + + + pr/ From 7284d22dff9a509db707a0fb01255508969928a2 Mon Sep 17 00:00:00 2001 From: Tomas Date: Mon, 2 Dec 2019 12:47:13 +0100 Subject: [PATCH 12/25] Bump up msbuild diagnostics for the sendtohelix script --- eng/pipelines/libraries/helix.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/libraries/helix.yml b/eng/pipelines/libraries/helix.yml index 945b51bfe72635..1730baa542daf0 100644 --- a/eng/pipelines/libraries/helix.yml +++ b/eng/pipelines/libraries/helix.yml @@ -13,6 +13,7 @@ steps: - script: ${{ parameters.msbuildScript }} $(Build.SourcesDirectory)/src/libraries/sendtohelix.proj /t:test + /v:diag /p:ArchGroup=${{ parameters.archGroup }} /p:ConfigurationGroup=${{ parameters.configuration }} /p:OSGroup=${{ parameters.targetOS }} From daa935d9f073d691eb5b7018006cf3b944eeeee2 Mon Sep 17 00:00:00 2001 From: Tomas Date: Mon, 2 Dec 2019 13:45:43 +0100 Subject: [PATCH 13/25] Fix configuration ID in calls to helix.yml --- eng/pipelines/libraries/build-job.yml | 2 +- eng/pipelines/libraries/run-test-job.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index 01f343642cf240..06cd54257b1eaa 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -128,7 +128,7 @@ jobs: parameters: targetOS: ${{ parameters.osGroup }} archGroup: ${{ parameters.archType }} - configuration: ${{ parameters.buildConfig }} + configuration: $(_BuildConfig) helixQueues: ${{ join('+', parameters.helixQueues) }} msbuildScript: $(_msbuildCommand) framework: ${{ parameters.framework }} diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index ffb0accf56d663..f75f218bd6e5ee 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -58,7 +58,7 @@ jobs: parameters: targetOS: ${{ parameters.osGroup }} archGroup: ${{ parameters.archType }} - configuration: ${{ parameters.buildConfig }} + configuration: $(_BuildConfig) helixQueues: ${{ join('+', parameters.helixQueues) }} msbuildScript: $(_msbuildCommand) framework: ${{ parameters.framework }} From 052313e126e1fbb9449d65dee651942f8c01a127 Mon Sep 17 00:00:00 2001 From: Tomas Date: Mon, 2 Dec 2019 16:05:53 +0100 Subject: [PATCH 14/25] Remove instrumentation; hopefully fix the coreclr-libraries pipeline --- eng/pipelines/coreclr/pr.yml | 1 - eng/pipelines/libraries/build-job.yml | 2 +- src/libraries/sendtohelix.proj | 11 ----------- 3 files changed, 1 insertion(+), 13 deletions(-) diff --git a/eng/pipelines/coreclr/pr.yml b/eng/pipelines/coreclr/pr.yml index 4681be90536d67..e4c5ff8f8d1aef 100644 --- a/eng/pipelines/coreclr/pr.yml +++ b/eng/pipelines/coreclr/pr.yml @@ -111,7 +111,6 @@ jobs: - Linux_x64 - Windows_NT_x86 - Windows_NT_x64 - - Windows_NT_arm64 helixQueueGroup: pr helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml jobParameters: diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index 06cd54257b1eaa..6c257c2e7299ea 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -44,7 +44,7 @@ jobs: - EMSDK_PATH: $(Build.BinariesDirectory)/emsdk - ${{ if eq(parameters.runTests, true) }}: - _buildAction: -build -buildtests /p:ArchiveTests=true - - ${{ if and(ne(parameters.framework, 'allConfigurations'), not(parameters.useLiveCoreClr)) }}: + - ${{ if and(ne(parameters.framework, 'allConfigurations'), ne(parameters.useLiveCoreClr, true)) }}: - _skipTestRestoreArg: /p:SkipTestRestore=false - ${{ parameters.variables }} diff --git a/src/libraries/sendtohelix.proj b/src/libraries/sendtohelix.proj index 426386341ffd11..ed0514b2de74b4 100644 --- a/src/libraries/sendtohelix.proj +++ b/src/libraries/sendtohelix.proj @@ -1,7 +1,4 @@ - - - pr/ @@ -87,9 +84,6 @@ Outputs="$(TestArchiveRuntimeFile)" Condition="'$(TargetGroup)' != 'AllConfigurations'"> - - - - - - - - From 3b23c9aa644a24f47580ed5c1ac206835dbba06a Mon Sep 17 00:00:00 2001 From: Tomas Date: Mon, 2 Dec 2019 22:05:58 +0100 Subject: [PATCH 15/25] Addressing most of PR feedback --- eng/pipelines/coreclr/pr.yml | 167 +++++++++++++++++++------- eng/pipelines/libraries/base-job.yml | 4 - eng/pipelines/libraries/build-job.yml | 2 +- eng/pipelines/root.yml | 123 +++++++++++++++++++ src/coreclr/build-test.cmd | 4 +- src/coreclr/build-test.sh | 6 +- 6 files changed, 254 insertions(+), 52 deletions(-) create mode 100644 eng/pipelines/root.yml diff --git a/eng/pipelines/coreclr/pr.yml b/eng/pipelines/coreclr/pr.yml index e4c5ff8f8d1aef..360b58dcb52bbd 100644 --- a/eng/pipelines/coreclr/pr.yml +++ b/eng/pipelines/coreclr/pr.yml @@ -4,6 +4,24 @@ pr: branches: include: - master + - release/*.* + paths: + include: + - '*' + - src/libraries/System.Private.CoreLib/* + exclude: + - docs/* + - CODE-OF-CONDUCT.md + - CONTRIBUTING.md + - LICENSE.TXT + - PATENTS.TXT + - README.md + - SECURITY.md + - THIRD-PARTY-NOTICES.TXT + - src/installer/* + - src/libraries/* + - eng/pipelines/installer/* + - eng/pipelines/libraries/* jobs: # @@ -12,111 +30,176 @@ jobs: - template: /eng/pipelines/common/checkout-job.yml # -# Build CoreCLR +# Debug builds # - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml - buildConfig: checked + buildConfig: debug platforms: - - OSX_x64 - - Linux_x64 - - Windows_NT_x86 - Windows_NT_x64 - - Windows_NT_arm64 + - Windows_NT_x86 jobParameters: testGroup: innerloop # -# Build CoreFX using live CoreLib from CoreCLR +# Checked builds # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml + jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml buildConfig: checked platforms: - - OSX_x64 + - Linux_arm + - Linux_arm64 + - Linux_musl_x64 - Linux_x64 - - Windows_NT_x86 + - OSX_x64 + - Windows_NT_arm + - Windows_NT_arm64 - Windows_NT_x64 + - Windows_NT_x86 + jobParameters: + testGroup: innerloop + +# +# Release builds +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml + buildConfig: release + platforms: + - Linux_arm64 + - Linux_musl_x64 + - Linux_rhel6_x64 + - OSX_x64 + - Windows_NT_arm - Windows_NT_arm64 + - Windows_NT_x64 jobParameters: - isOfficialBuild: false - testScope: empty - useLiveCoreClr: true + testGroup: innerloop # -# Test builds using live CoreFX +# Checked test builds # - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/coreclr/templates/build-test-job.yml buildConfig: checked platforms: + - Linux_arm + - Linux_arm64 - OSX_x64 - - Linux_x64 - - Windows_NT_x86 - - Windows_NT_x64 + - Windows_NT_arm - Windows_NT_arm64 + - Windows_NT_x64 + - Windows_NT_x86 jobParameters: testGroup: innerloop - useLiveLibraries: true # -# CoreCLR Test executions +# Checked JIT test executions # - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/coreclr/templates/run-test-job.yml buildConfig: checked platforms: - - OSX_x64 + - Linux_arm + - Linux_arm64 + - Linux_musl_x64 - Linux_x64 - - Windows_NT_x86 - - Windows_NT_x64 + - OSX_x64 + - Windows_NT_arm - Windows_NT_arm64 + - Windows_NT_x64 + - Windows_NT_x86 helixQueueGroup: pr helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: testGroup: innerloop - useLiveLibraries: true # -# CoreFX Test Build +# Checked R2R test executions # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/libraries/build-test-job.yml + jobTemplate: /eng/pipelines/coreclr/templates/run-test-job.yml buildConfig: checked platforms: - - OSX_x64 - Linux_x64 - - Windows_NT_x86 + - OSX_x64 - Windows_NT_x64 - - Windows_NT_arm64 + - Windows_NT_x86 + helixQueueGroup: pr + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: - isOfficialBuild: false - testScope: innerloop - useLiveCoreClr: true + testGroup: innerloop + readyToRun: true + displayNameArgs: R2R # -# CoreFX Test Execution +# CoreFX test runs against CoreCLR # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml + jobTemplate: /eng/pipelines/coreclr/templates/run-test-job.yml buildConfig: checked platforms: - - OSX_x64 - Linux_x64 - - Windows_NT_x86 - Windows_NT_x64 helixQueueGroup: pr - helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: innerloop + corefxTests: true + displayNameArgs: CoreFX + +# +# Crossgen-comparison jobs +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/crossgen-comparison-job.yml + buildConfig: checked + platforms: + - Linux_arm + helixQueueGroup: pr + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + +# +# Release test builds +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-test-job.yml + buildConfig: release + platforms: + - OSX_x64 + jobParameters: + testGroup: innerloop + +# +# Release test builds +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/run-test-job.yml + buildConfig: release + platforms: + - Linux_musl_x64 + helixQueueGroup: pr + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: - isOfficialBuild: false - isFullMatrix: false - framework: netcoreapp - testScope: innerloop - useLiveCoreClr: true - dependsOnBuildConfiguration: checked + testGroup: innerloop + +# +# Formatting +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/format-job.yml + platforms: + - Linux_x64 + - Windows_NT_x64 diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index 4502cc36d17c68..5cc15374935696 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -80,15 +80,11 @@ jobs: # Windows variables - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - _msbuildCommand: powershell -ExecutionPolicy ByPass -NoProfile eng\common\msbuild.ps1 -warnaserror:0 -ci - _runtimeOSArg: /p:RuntimeOS=win10 - - _archiveExtension: '.zip' # Non-Windows variables - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - _msbuildCommand: ./eng/common/msbuild.sh --warnaserror false --ci - _buildScript: ./$(buildScriptFileName)$(scriptExt) - - _archiveExtension: '.tar.gz' - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - _stripSymbolsArg: -stripSymbols diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index 6c257c2e7299ea..f5636a39e0ec22 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -44,7 +44,7 @@ jobs: - EMSDK_PATH: $(Build.BinariesDirectory)/emsdk - ${{ if eq(parameters.runTests, true) }}: - _buildAction: -build -buildtests /p:ArchiveTests=true - - ${{ if and(ne(parameters.framework, 'allConfigurations'), ne(parameters.useLiveCoreClr, true)) }}: + - ${{ if ne(parameters.framework, 'allConfigurations') }}: - _skipTestRestoreArg: /p:SkipTestRestore=false - ${{ parameters.variables }} diff --git a/eng/pipelines/root.yml b/eng/pipelines/root.yml new file mode 100644 index 00000000000000..082cc7f6f50c3d --- /dev/null +++ b/eng/pipelines/root.yml @@ -0,0 +1,123 @@ +trigger: none + +pr: + branches: + include: + - master + - release/*.* + +jobs: +# +# Checkout repository +# +- template: /eng/pipelines/common/checkout-job.yml + +# +# Build CoreCLR +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml + buildConfig: checked + platforms: + - OSX_x64 + - Linux_x64 + - Windows_NT_x86 + - Windows_NT_x64 + - Windows_NT_arm64 + jobParameters: + testGroup: innerloop + +# +# Build CoreFX using live CoreLib from CoreCLR +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build-job.yml + buildConfig: checked + platforms: + - OSX_x64 + - Linux_x64 + - Windows_NT_x86 + - Windows_NT_x64 + - Windows_NT_arm64 + jobParameters: + isOfficialBuild: false + testScope: empty + useLiveCoreClr: true + +# +# Test builds using live CoreFX +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-test-job.yml + buildConfig: checked + platforms: + - OSX_x64 + - Linux_x64 + - Windows_NT_x86 + - Windows_NT_x64 + - Windows_NT_arm64 + jobParameters: + testGroup: innerloop + useLiveLibraries: true + +# +# CoreCLR Test executions +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/run-test-job.yml + buildConfig: checked + platforms: + - OSX_x64 + - Linux_x64 + - Windows_NT_x86 + - Windows_NT_x64 + - Windows_NT_arm64 + helixQueueGroup: pr + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: innerloop + useLiveLibraries: true + +# +# CoreFX Test Build +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build-test-job.yml + buildConfig: checked + platforms: + - OSX_x64 + - Linux_x64 + - Windows_NT_x86 + - Windows_NT_x64 + - Windows_NT_arm64 + jobParameters: + isOfficialBuild: false + testScope: innerloop + useLiveCoreClr: true + +# +# CoreFX Test Execution +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/run-test-job.yml + buildConfig: checked + platforms: + - OSX_x64 + - Linux_x64 + - Windows_NT_x86 + - Windows_NT_x64 + helixQueueGroup: pr + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + jobParameters: + isOfficialBuild: false + isFullMatrix: false + framework: netcoreapp + testScope: innerloop + useLiveCoreClr: true + dependsOnBuildConfiguration: checked diff --git a/src/coreclr/build-test.cmd b/src/coreclr/build-test.cmd index dfec36661d5c35..0a078d2935f1b0 100644 --- a/src/coreclr/build-test.cmd +++ b/src/coreclr/build-test.cmd @@ -581,8 +581,8 @@ REM ============================================================================ if NOT "%__LocalCoreFXPath%"=="" ( echo Patch CoreFX from %__LocalCoreFXPath% - set __CoreFXBuildType=Debug - if "%__BuildType%" == "release" (set __CoreFXBuildType=Release) + set __CoreFXBuildType=Release + if "%__BuildType%" == "debug" (set __CoreFXBuildType=Debug) set NEXTCMD=python "%__ProjectDir%\tests\scripts\patch-corefx.py" -clr_core_root "%CORE_ROOT%"^ -fx_root "%__LocalCoreFXPath%" -arch %__BuildArch% -build_type !__CoreFXBuildType! echo !NEXTCMD! diff --git a/src/coreclr/build-test.sh b/src/coreclr/build-test.sh index f0abe91289a32f..97ee28bfdc36f1 100755 --- a/src/coreclr/build-test.sh +++ b/src/coreclr/build-test.sh @@ -136,9 +136,9 @@ patch_corefx_libraries() { echo "${__MsgPrefix}Patching CORE_ROOT: '${CORE_ROOT}' with CoreFX libaries from enlistment '${__LocalCoreFXPath}" - __CoreFXBuildType=Debug - if [ "$__BuildType" == "release" ]; then - __CoreFXBuildType=Release + __CoreFXBuildType=Release + if [ "$__BuildType" == "debug" ]; then + __CoreFXBuildType=Debug fi patchCoreFXArguments=("-clr_core_root" "${CORE_ROOT}" "-fx_root" "${__LocalCoreFXPath}" "-arch" "${__BuildArch}" "-build_type" "${__CoreFXBuildType}") From 58f8c52019a2c0b55473889eb3d3ea76365379bd Mon Sep 17 00:00:00 2001 From: Tomas Date: Mon, 2 Dec 2019 22:37:08 +0100 Subject: [PATCH 16/25] Fix unification of archiveExtension --- eng/pipelines/libraries/base-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index 5cc15374935696..38a0d8aca5e64e 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -117,7 +117,7 @@ jobs: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(_coreClrDownloadPath) - artifactFileName: '$(_coreClrArtifactName)$(_archiveExtension)' + artifactFileName: '$(_coreClrArtifactName)$(archiveExtension)' artifactName: '$(_coreClrArtifactName)' displayName: 'CoreCLR build drop' From b8d12646cb68cf16ed9eb6d868804aa2a7d675df Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 3 Dec 2019 08:30:04 +0100 Subject: [PATCH 17/25] Santiago's feedback - rename coreFx to libraries in the new code --- eng/pipelines/coreclr/templates/build-test-job.yml | 10 +++++----- eng/pipelines/coreclr/templates/run-test-job.yml | 14 +++++++------- .../coreclr/templates/xplat-pipeline-job.yml | 14 +++++++------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/eng/pipelines/coreclr/templates/build-test-job.yml b/eng/pipelines/coreclr/templates/build-test-job.yml index d9175e7b66c379..7b61ce17d65888 100644 --- a/eng/pipelines/coreclr/templates/build-test-job.yml +++ b/eng/pipelines/coreclr/templates/build-test-job.yml @@ -81,14 +81,14 @@ jobs: displayName: Install native dependencies - # Optionally download live-built CoreFX artifacts + # Optionally download live-built libraries - ${{ if eq(parameters.useLiveLibraries, true) }}: - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: $(coreFxDownloadDir) + unpackFolder: $(librariesDownloadDir) cleanUnpackFolder: false - artifactFileName: '$(coreFxArtifactName)$(archiveExtension)' - artifactName: '$(coreFxArtifactName)' + artifactFileName: '$(librariesArtifactName)$(archiveExtension)' + artifactName: '$(librariesArtifactName)' displayName: 'live-built CoreFX' @@ -102,7 +102,7 @@ jobs: # Build managed test components - - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipnative skipgeneratelayout $(buildConfig) $(archType) $(crossArg) $(priorityArg) ci $(coreFxOverrideArg) + - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipnative skipgeneratelayout $(buildConfig) $(archType) $(crossArg) $(priorityArg) ci $(librariesOverrideArg) displayName: Build managed test components diff --git a/eng/pipelines/coreclr/templates/run-test-job.yml b/eng/pipelines/coreclr/templates/run-test-job.yml index 461bef23a3f997..b3fe0abc901e1f 100644 --- a/eng/pipelines/coreclr/templates/run-test-job.yml +++ b/eng/pipelines/coreclr/templates/run-test-job.yml @@ -107,14 +107,14 @@ jobs: steps: - # Optionally download live-built CoreFX artifacts + # Optionally download live-built libraries - ${{ if eq(parameters.useLiveLibraries, true) }}: - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: $(coreFxDownloadDir) + unpackFolder: $(librariesDownloadDir) cleanUnpackFolder: false - artifactFileName: '$(coreFxArtifactName)$(archiveExtension)' - artifactName: '$(coreFxArtifactName)' + artifactFileName: '$(librariesArtifactName)$(archiveExtension)' + artifactName: '$(librariesArtifactName)' displayName: 'live-built CoreFX' @@ -164,19 +164,19 @@ jobs: # the native artifacts to the final test folders is dependent on availability of the # managed test artifacts. - ${{ if ne(parameters.corefxTests, true) }}: - - script: $(coreClrRepoRootDir)build-test$(scriptExt) copynativeonly $(buildConfig) $(archType) $(priorityArg) $(coreFxOverrideArg) + - script: $(coreClrRepoRootDir)build-test$(scriptExt) copynativeonly $(buildConfig) $(archType) $(priorityArg) $(librariesOverrideArg) displayName: Copy native test components to test output folder # Generate test host - ${{ if eq(parameters.corefxTests, true) }}: - - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipmanaged skipnative $(testhostArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(coreFxOverrideArg) + - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipmanaged skipnative $(testhostArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(librariesOverrideArg) displayName: Generate test host # Crossgen framework assemblies prior to triggering readyToRun execution runs. - ${{ if eq(parameters.readyToRun, true) }}: - - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipmanaged skipnative $(crossgenArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(coreFxOverrideArg) + - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipmanaged skipnative $(crossgenArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(librariesOverrideArg) displayName: Crossgen framework assemblies diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index de2cda2199f32a..930e9040be7a8f 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -104,17 +104,17 @@ jobs: - name: priorityArg value: '-priority=1' - - coreFxArtifactName: '' - - coreFxOverrideArg: '' - - coreFxDownloadDir: '' + - librariesArtifactName: '' + - librariesOverrideArg: '' + - librariesDownloadDir: '' - ${{ if eq(parameters.useLiveLibraries, true) }}: - - coreFxArtifactName: ${{ format('libraries_bin_{0}_{1}{2}_{3}_{4}', 'netcoreapp', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - coreFxDownloadDir: $(Build.SourcesDirectory)/artifacts + - librariesArtifactName: ${{ format('libraries_bin_{0}_{1}{2}_{3}_{4}', 'netcoreapp', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - librariesDownloadDir: $(Build.SourcesDirectory)/artifacts - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - coreFxOverrideArg: ' localcorefxpath $(Build.SourcesDirectory)' + - librariesOverrideArg: ' localcorefxpath $(Build.SourcesDirectory)' - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - coreFxOverrideArg: ' localcorefxpath=$(Build.SourcesDirectory)' + - librariesOverrideArg: ' localcorefxpath=$(Build.SourcesDirectory)' - ${{ each variable in parameters.variables }}: - ${{insert}}: ${{ variable }} From f69fca75eff00b913d61fe2d3a67336e01e2e153 Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 3 Dec 2019 08:40:36 +0100 Subject: [PATCH 18/25] Introduce liveCoreClrBuildConfig and liveLibrariesBuildConfig These complementary properties should let us drop all automatic transformations of CoreCLR vs. library build modes and thus substantially simplify the live-live pipeline and make it more general at the same time. I have reverted the changes to the build-test scripts that were taking care of this logic on the local build side as well as the change to eng/pipelines/libraries/base-job and bits of the other YAML files on the CI side. Thanks Tomas --- eng/pipelines/coreclr/templates/build-test-job.yml | 10 +++++----- eng/pipelines/coreclr/templates/run-test-job.yml | 6 +++--- .../coreclr/templates/xplat-pipeline-job.yml | 6 +++--- eng/pipelines/libraries/base-job.yml | 12 +++++------- eng/pipelines/libraries/build-job.yml | 10 +++++----- eng/pipelines/libraries/build-test-job.yml | 4 ++-- eng/pipelines/libraries/run-test-job.yml | 6 +++--- eng/pipelines/root.yml | 14 +++++++------- src/coreclr/build-test.cmd | 4 +--- src/coreclr/build-test.sh | 7 +------ 10 files changed, 35 insertions(+), 44 deletions(-) diff --git a/eng/pipelines/coreclr/templates/build-test-job.yml b/eng/pipelines/coreclr/templates/build-test-job.yml index 7b61ce17d65888..74a39228bb9aab 100644 --- a/eng/pipelines/coreclr/templates/build-test-job.yml +++ b/eng/pipelines/coreclr/templates/build-test-job.yml @@ -5,7 +5,7 @@ parameters: osSubgroup: '' container: '' testGroup: '' - useLiveLibraries: false + liveLibrariesBuildConfig: '' displayNameArgs: '' condition: true stagedBuild: false @@ -32,7 +32,7 @@ jobs: container: ${{ parameters.container }} testGroup: ${{ parameters.testGroup }} stagedBuild: ${{ parameters.stagedBuild }} - useLiveLibraries: ${{ parameters.useLiveLibraries }} + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} variables: ${{ parameters.variables }} pool: ${{ parameters.pool }} @@ -59,8 +59,8 @@ jobs: ${{ if ne(parameters.stagedBuild, true) }}: dependsOn: - ${{ format('coreclr_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if eq(parameters.useLiveLibraries, true) }}: - - ${{ format('libraries_build_{0}_{1}{2}_{3}_{4}', 'netcoreapp', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: + - ${{ format('libraries_build_{0}_{1}{2}_{3}_{4}', 'netcoreapp', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} ${{ if eq(parameters.testGroup, 'innerloop') }}: @@ -82,7 +82,7 @@ jobs: # Optionally download live-built libraries - - ${{ if eq(parameters.useLiveLibraries, true) }}: + - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(librariesDownloadDir) diff --git a/eng/pipelines/coreclr/templates/run-test-job.yml b/eng/pipelines/coreclr/templates/run-test-job.yml index b3fe0abc901e1f..70ed97007b5a19 100644 --- a/eng/pipelines/coreclr/templates/run-test-job.yml +++ b/eng/pipelines/coreclr/templates/run-test-job.yml @@ -9,7 +9,7 @@ parameters: testGroup: '' crossrootfsDir: '' readyToRun: false - useLiveLibraries: false + liveLibrariesBuildConfig: '' crossgen2: false helixQueues: '' # If true, run the corefx tests instead of the coreclr ones @@ -39,7 +39,7 @@ jobs: crossrootfsDir: ${{ parameters.crossrootfsDir }} corefxTests: ${{ parameters.corefxTests }} stagedBuild: ${{ parameters.stagedBuild }} - useLiveLibraries: ${{ parameters.useLiveLibraries }} + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} helixType: 'build/tests/' pool: ${{ parameters.pool }} @@ -108,7 +108,7 @@ jobs: steps: # Optionally download live-built libraries - - ${{ if eq(parameters.useLiveLibraries, true) }}: + - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(librariesDownloadDir) diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index 930e9040be7a8f..1c210be06ebb21 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -11,7 +11,7 @@ parameters: testGroup: '' crossrootfsDir: '' corefxTests: false - useLiveLibraries: false + liveLibrariesBuildConfig: ''; stagedBuild: false strategy: '' pool: '' @@ -108,8 +108,8 @@ jobs: - librariesOverrideArg: '' - librariesDownloadDir: '' - - ${{ if eq(parameters.useLiveLibraries, true) }}: - - librariesArtifactName: ${{ format('libraries_bin_{0}_{1}{2}_{3}_{4}', 'netcoreapp', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: + - librariesArtifactName: ${{ format('libraries_bin_{0}_{1}{2}_{3}_{4}', 'netcoreapp', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} - librariesDownloadDir: $(Build.SourcesDirectory)/artifacts - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - librariesOverrideArg: ' localcorefxpath $(Build.SourcesDirectory)' diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index 38a0d8aca5e64e..2108bf992a6fe5 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -5,7 +5,7 @@ parameters: osSubgroup: '' framework: '' isOfficialBuild: false - useLiveCoreClr: false + liveCoreClrBuildConfig: '' timeoutInMinutes: 150 container: '' steps: [] @@ -31,9 +31,7 @@ jobs: pool: ${{ parameters.pool }} variables: - buildScriptFileName: libraries - - _BuildConfig: Debug - - ${{ if eq(parameters.buildConfig, 'release') }}: - - _BuildConfig: Release + - _BuildConfig: ${{ parameters.buildConfig }} - _msbuildCommonParameters: '' - _stripSymbolsArg: '' @@ -73,9 +71,9 @@ jobs: - _coreClrDownloadPath: '' - _coreClrOverridePathArg: '' - - ${{ if eq(parameters.useLiveCoreClr, true) }}: + - ${{ if ne(parameters.liveCoreClrBuildConfig, '') }}: - _coreClrDownloadPath: '$(Build.SourcesDirectory)/artifacts/transport/coreclr' - - _coreClrArtifactName: 'CoreCLRProduct_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}' + - _coreClrArtifactName: 'CoreCLRProduct_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveCoreClrBuildConfig }}' - _coreClrOverridePathArg: ' /p:CoreCLROverridePath=$(_coreClrDownloadPath)' # Windows variables @@ -88,7 +86,7 @@ jobs: - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - _stripSymbolsArg: -stripSymbols - - _buildArguments: -configuration $(_BuildConfig) -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) $(_stripSymbolsArg) $(_testScopeArg) $(_warnAsErrorArg) $(_runtimeOSArg) $(_portableArg) $(_msbuildCommonParameters) + - _buildArguments: -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) $(_stripSymbolsArg) $(_testScopeArg) $(_warnAsErrorArg) $(_runtimeOSArg) $(_portableArg) $(_msbuildCommonParameters) - ${{ parameters.variables }} dependsOn: diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index f5636a39e0ec22..3947baec048c52 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -5,7 +5,7 @@ parameters: archType: '' framework: netcoreapp isOfficialBuild: false - useLiveCoreClr: false + liveCoreClrBuildConfig: '' timeoutInMinutes: 150 preBuildSteps: [] container: '' @@ -23,7 +23,7 @@ jobs: archType: ${{ parameters.archType }} framework: ${{ parameters.framework }} isOfficialBuild: ${{ parameters.isOfficialBuild }} - useLiveCoreClr: ${{ parameters.useLiveCoreClr }} + liveCoreClrBuildConfig: ${{ parameters.liveCoreClrBuildConfig }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} preBuildSteps: ${{ parameters.preBuildSteps }} container: ${{ parameters.container }} @@ -32,9 +32,9 @@ jobs: name: build displayName: 'Build' - ${{ if eq(parameters.useLiveCoreClr, true) }}: + ${{ if ne(parameters.liveCoreClrBuildConfig, '') }}: dependsOn: - - ${{ format('coreclr_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ format('coreclr_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveCoreClrBuildConfig) }} variables: - _skipTestRestoreArg: /p:SkipTestRestore=true @@ -128,7 +128,7 @@ jobs: parameters: targetOS: ${{ parameters.osGroup }} archGroup: ${{ parameters.archType }} - configuration: $(_BuildConfig) + configuration: ${{ parameters.buildConfig }} helixQueues: ${{ join('+', parameters.helixQueues) }} msbuildScript: $(_msbuildCommand) framework: ${{ parameters.framework }} diff --git a/eng/pipelines/libraries/build-test-job.yml b/eng/pipelines/libraries/build-test-job.yml index bfa312b54023ad..0857e2f300686e 100644 --- a/eng/pipelines/libraries/build-test-job.yml +++ b/eng/pipelines/libraries/build-test-job.yml @@ -5,7 +5,7 @@ parameters: archType: '' framework: netcoreapp isOfficialBuild: false - useLiveCoreClr: false + liveCoreClrBuildConfig: '' timeoutInMinutes: 150 container: '' publishTestArtifacs: true @@ -23,7 +23,7 @@ jobs: archType: ${{ parameters.archType }} framework: ${{ parameters.framework }} isOfficialBuild: ${{ parameters.isOfficialBuild }} - useLiveCoreClr: ${{ parameters.useLiveCoreClr }} + liveCoreClrBuildConfig: ${{ parameters.liveCoreClrBuildConfig }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} container: ${{ parameters.container }} pool: ${{ parameters.pool }} diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index f75f218bd6e5ee..06b2a7b47254ba 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -5,7 +5,7 @@ parameters: archType: '' framework: netcoreapp isOfficialBuild: false - useLiveCoreClr: false + liveCoreClrBuildConfig: '' timeoutInMinutes: 150 pool: '' testScope: '' @@ -22,7 +22,7 @@ jobs: archType: ${{ parameters.archType }} framework: ${{ parameters.framework }} isOfficialBuild: ${{ parameters.isOfficialBuild }} - useLiveCoreClr: ${{ parameters.useLiveCoreClr }} + liveCoreClrBuildConfig: ${{ parameters.liveCoreClrBuildConfig }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} container: '' # we just send to helix, no need to use a container. pool: ${{ parameters.pool }} @@ -58,7 +58,7 @@ jobs: parameters: targetOS: ${{ parameters.osGroup }} archGroup: ${{ parameters.archType }} - configuration: $(_BuildConfig) + configuration: ${{ parameters.buildConfig }} helixQueues: ${{ join('+', parameters.helixQueues) }} msbuildScript: $(_msbuildCommand) framework: ${{ parameters.framework }} diff --git a/eng/pipelines/root.yml b/eng/pipelines/root.yml index 082cc7f6f50c3d..900fdf1ffac439 100644 --- a/eng/pipelines/root.yml +++ b/eng/pipelines/root.yml @@ -34,7 +34,7 @@ jobs: - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/libraries/build-job.yml - buildConfig: checked + buildConfig: Release platforms: - OSX_x64 - Linux_x64 @@ -44,7 +44,7 @@ jobs: jobParameters: isOfficialBuild: false testScope: empty - useLiveCoreClr: true + liveCoreClrBuildConfig: checked # # Test builds using live CoreFX @@ -61,7 +61,7 @@ jobs: - Windows_NT_arm64 jobParameters: testGroup: innerloop - useLiveLibraries: true + liveLibrariesBuildConfig: Debug # # CoreCLR Test executions @@ -80,7 +80,7 @@ jobs: helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: testGroup: innerloop - useLiveLibraries: true + liveLibrariesBuildConfig: Debug # # CoreFX Test Build @@ -88,7 +88,7 @@ jobs: - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/libraries/build-test-job.yml - buildConfig: checked + buildConfig: Release platforms: - OSX_x64 - Linux_x64 @@ -98,7 +98,7 @@ jobs: jobParameters: isOfficialBuild: false testScope: innerloop - useLiveCoreClr: true + liveCoreClrBuildConfig: checked # # CoreFX Test Execution @@ -106,7 +106,7 @@ jobs: - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: checked + buildConfig: Release platforms: - OSX_x64 - Linux_x64 diff --git a/src/coreclr/build-test.cmd b/src/coreclr/build-test.cmd index 0a078d2935f1b0..3680d37c30768b 100644 --- a/src/coreclr/build-test.cmd +++ b/src/coreclr/build-test.cmd @@ -581,10 +581,8 @@ REM ============================================================================ if NOT "%__LocalCoreFXPath%"=="" ( echo Patch CoreFX from %__LocalCoreFXPath% - set __CoreFXBuildType=Release - if "%__BuildType%" == "debug" (set __CoreFXBuildType=Debug) set NEXTCMD=python "%__ProjectDir%\tests\scripts\patch-corefx.py" -clr_core_root "%CORE_ROOT%"^ - -fx_root "%__LocalCoreFXPath%" -arch %__BuildArch% -build_type !__CoreFXBuildType! + -fx_root "%__LocalCoreFXPath%" -arch %__BuildArch% -build_type %__BuildType% echo !NEXTCMD! !NEXTCMD! ) diff --git a/src/coreclr/build-test.sh b/src/coreclr/build-test.sh index 97ee28bfdc36f1..65eb921d53fc89 100755 --- a/src/coreclr/build-test.sh +++ b/src/coreclr/build-test.sh @@ -136,12 +136,7 @@ patch_corefx_libraries() { echo "${__MsgPrefix}Patching CORE_ROOT: '${CORE_ROOT}' with CoreFX libaries from enlistment '${__LocalCoreFXPath}" - __CoreFXBuildType=Release - if [ "$__BuildType" == "debug" ]; then - __CoreFXBuildType=Debug - fi - - patchCoreFXArguments=("-clr_core_root" "${CORE_ROOT}" "-fx_root" "${__LocalCoreFXPath}" "-arch" "${__BuildArch}" "-build_type" "${__CoreFXBuildType}") + patchCoreFXArguments=("-clr_core_root" "${CORE_ROOT}" "-fx_root" "${__LocalCoreFXPath}" "-arch" "${__BuildArch}" "-build_type" "${__BuildType}") scriptPath="$__ProjectDir/tests/scripts" echo "python ${scriptPath}/patch-corefx.py ${patchCoreFXArguments[@]}" $__Python "${scriptPath}/patch-corefx.py" "${patchCoreFXArguments[@]}" From 91ce5a4b6f7c2334f6aa686b2b94cb2ac7b348b7 Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 3 Dec 2019 08:44:04 +0100 Subject: [PATCH 19/25] Rename root.yml to runtime.yml per PR feedback --- eng/pipelines/coreclr/templates/xplat-pipeline-job.yml | 2 +- eng/pipelines/libraries/base-job.yml | 2 +- eng/pipelines/{root.yml => runtime.yml} | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) rename eng/pipelines/{root.yml => runtime.yml} (94%) diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index 1c210be06ebb21..5f897c3c586bf7 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -11,7 +11,7 @@ parameters: testGroup: '' crossrootfsDir: '' corefxTests: false - liveLibrariesBuildConfig: ''; + liveLibrariesBuildConfig: '' stagedBuild: false strategy: '' pool: '' diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index 2108bf992a6fe5..5f14b09c9c1bcb 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -111,7 +111,7 @@ jobs: git remote set-url origin $(Build.Repository.Uri) displayName: Clone the repository from Checkout.bundle - - ${{ if eq(parameters.useLiveCoreClr, true) }}: + - ${{ if ne(parameters.liveCoreClrBuildConfig, '') }}: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(_coreClrDownloadPath) diff --git a/eng/pipelines/root.yml b/eng/pipelines/runtime.yml similarity index 94% rename from eng/pipelines/root.yml rename to eng/pipelines/runtime.yml index 900fdf1ffac439..13f61f7d41b4e9 100644 --- a/eng/pipelines/root.yml +++ b/eng/pipelines/runtime.yml @@ -61,7 +61,7 @@ jobs: - Windows_NT_arm64 jobParameters: testGroup: innerloop - liveLibrariesBuildConfig: Debug + liveLibrariesBuildConfig: Release # # CoreCLR Test executions @@ -80,7 +80,7 @@ jobs: helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: testGroup: innerloop - liveLibrariesBuildConfig: Debug + liveLibrariesBuildConfig: Release # # CoreFX Test Build @@ -119,5 +119,5 @@ jobs: isFullMatrix: false framework: netcoreapp testScope: innerloop - useLiveCoreClr: true - dependsOnBuildConfiguration: checked + liveCoreClrBuildConfig: checked + dependsOnBuildConfiguration: Release From b1bf69bb50bd36750e199803ad9c262977dce4fb Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 3 Dec 2019 13:46:20 +0100 Subject: [PATCH 20/25] Add build-test argument to specify CoreFX configuration --- eng/pipelines/coreclr/templates/run-test-job.yml | 2 +- eng/pipelines/coreclr/templates/xplat-pipeline-job.yml | 4 ++-- src/coreclr/build-test.cmd | 6 ++++-- src/coreclr/build-test.sh | 10 ++++++++-- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/eng/pipelines/coreclr/templates/run-test-job.yml b/eng/pipelines/coreclr/templates/run-test-job.yml index 70ed97007b5a19..f2e8da1eb75007 100644 --- a/eng/pipelines/coreclr/templates/run-test-job.yml +++ b/eng/pipelines/coreclr/templates/run-test-job.yml @@ -115,7 +115,7 @@ jobs: cleanUnpackFolder: false artifactFileName: '$(librariesArtifactName)$(archiveExtension)' artifactName: '$(librariesArtifactName)' - displayName: 'live-built CoreFX' + displayName: 'live-built libraries' # Download and unzip managed test artifacts diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index 5f897c3c586bf7..c3389f89fb8cf9 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -112,9 +112,9 @@ jobs: - librariesArtifactName: ${{ format('libraries_bin_{0}_{1}{2}_{3}_{4}', 'netcoreapp', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} - librariesDownloadDir: $(Build.SourcesDirectory)/artifacts - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - librariesOverrideArg: ' localcorefxpath $(Build.SourcesDirectory)' + - librariesOverrideArg: ' localcorefxpath $(Build.SourcesDirectory) localcorefxconfig $(parameters.liveLibrariesBuildConfig)' - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - librariesOverrideArg: ' localcorefxpath=$(Build.SourcesDirectory)' + - librariesOverrideArg: ' localcorefxpath=$(Build.SourcesDirectory) localcorefxconfig=$(parameters.liveLibrariesBuildConfig)' - ${{ each variable in parameters.variables }}: - ${{insert}}: ${{ variable }} diff --git a/src/coreclr/build-test.cmd b/src/coreclr/build-test.cmd index 3680d37c30768b..936fbfb3fc0901 100644 --- a/src/coreclr/build-test.cmd +++ b/src/coreclr/build-test.cmd @@ -60,6 +60,7 @@ set __CopyNativeTestBinaries=0 set __CopyNativeProjectsAfterCombinedTestBuild=true set __SkipGenerateLayout=0 set __LocalCoreFXPath= +set __LocalCoreFXConfig=%__BuildType% set __SkipFXRestoreArg= set __GenerateLayoutOnly=0 @@ -106,6 +107,7 @@ if /i "%1" == "-priority" (set __Priority=%2&shift&set processedArgs if /i "%1" == "copynativeonly" (set __CopyNativeTestBinaries=1&set __SkipNative=1&set __CopyNativeProjectsAfterCombinedTestBuild=false&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "skipgeneratelayout" (set __SkipGenerateLayout=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "localcorefxpath" (set __LocalCoreFXPath=%2&set __SkipFXRestoreArg=/p:__SkipFXRestore=true&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) +if /i "%1" == "localcorefxconfig" (set __LocalCoreFXConfig=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) if /i "%1" == "generatelayoutonly" (set __SkipManaged=1&set __SkipNative=1&set __CopyNativeProjectsAfterCombinedTestBuild=false&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "--" (set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) @@ -580,9 +582,9 @@ REM === REM ========================================================================================= if NOT "%__LocalCoreFXPath%"=="" ( - echo Patch CoreFX from %__LocalCoreFXPath% + echo Patch CoreFX from %__LocalCoreFXPath% (%__LocalCoreFXConfig%) set NEXTCMD=python "%__ProjectDir%\tests\scripts\patch-corefx.py" -clr_core_root "%CORE_ROOT%"^ - -fx_root "%__LocalCoreFXPath%" -arch %__BuildArch% -build_type %__BuildType% + -fx_root "%__LocalCoreFXPath%" -arch %__BuildArch% -build_type %__LocalCoreFXConfig% echo !NEXTCMD! !NEXTCMD! ) diff --git a/src/coreclr/build-test.sh b/src/coreclr/build-test.sh index 65eb921d53fc89..bd5a62064de8c0 100755 --- a/src/coreclr/build-test.sh +++ b/src/coreclr/build-test.sh @@ -134,9 +134,9 @@ generate_layout() patch_corefx_libraries() { - echo "${__MsgPrefix}Patching CORE_ROOT: '${CORE_ROOT}' with CoreFX libaries from enlistment '${__LocalCoreFXPath}" + echo "${__MsgPrefix}Patching CORE_ROOT: '${CORE_ROOT}' with CoreFX libaries from enlistment '${__LocalCoreFXPath} (${__LocalCoreFXConfig})" - patchCoreFXArguments=("-clr_core_root" "${CORE_ROOT}" "-fx_root" "${__LocalCoreFXPath}" "-arch" "${__BuildArch}" "-build_type" "${__BuildType}") + patchCoreFXArguments=("-clr_core_root" "${CORE_ROOT}" "-fx_root" "${__LocalCoreFXPath}" "-arch" "${__BuildArch}" "-build_type" "${__LocalCoreFXConfig}") scriptPath="$__ProjectDir/tests/scripts" echo "python ${scriptPath}/patch-corefx.py ${patchCoreFXArguments[@]}" $__Python "${scriptPath}/patch-corefx.py" "${patchCoreFXArguments[@]}" @@ -634,6 +634,10 @@ handle_arguments() { __LocalCoreFXPath=$(echo "$1" | cut -d'=' -f 2) ;; + localcorefxconfig=*|-localcorefxconfig=*) + __LocalCoreFXConfig=$(echo "$1" | cut -d'=' -f 2) + ;; + *) __UnprocessedBuildArgs+=("$1") ;; @@ -680,6 +684,8 @@ __SkipRestore="" __SkipRestorePackages=0 __SourceDir="$__ProjectDir/src" __UnprocessedBuildArgs= +__LocalCoreFXPath= +__LocalCoreFXConfig=${__BuildType} __UseNinja=0 __VerboseBuild=0 __cmakeargs="" From 5ec960713f34bbbb11f171d1b37500fddfe83c99 Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 3 Dec 2019 14:51:39 +0100 Subject: [PATCH 21/25] Remove msbuild verbosity instrumentation; fix YAML typo --- eng/pipelines/coreclr/templates/xplat-pipeline-job.yml | 4 ++-- eng/pipelines/libraries/helix.yml | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index c3389f89fb8cf9..b36cad5bf86bce 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -112,9 +112,9 @@ jobs: - librariesArtifactName: ${{ format('libraries_bin_{0}_{1}{2}_{3}_{4}', 'netcoreapp', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} - librariesDownloadDir: $(Build.SourcesDirectory)/artifacts - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - librariesOverrideArg: ' localcorefxpath $(Build.SourcesDirectory) localcorefxconfig $(parameters.liveLibrariesBuildConfig)' + - librariesOverrideArg: ' localcorefxpath $(Build.SourcesDirectory) localcorefxconfig ${{ parameters.liveLibrariesBuildConfig }}' - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - librariesOverrideArg: ' localcorefxpath=$(Build.SourcesDirectory) localcorefxconfig=$(parameters.liveLibrariesBuildConfig)' + - librariesOverrideArg: ' localcorefxpath=$(Build.SourcesDirectory) localcorefxconfig=${{ parameters.liveLibrariesBuildConfig }}' - ${{ each variable in parameters.variables }}: - ${{insert}}: ${{ variable }} diff --git a/eng/pipelines/libraries/helix.yml b/eng/pipelines/libraries/helix.yml index 1730baa542daf0..945b51bfe72635 100644 --- a/eng/pipelines/libraries/helix.yml +++ b/eng/pipelines/libraries/helix.yml @@ -13,7 +13,6 @@ steps: - script: ${{ parameters.msbuildScript }} $(Build.SourcesDirectory)/src/libraries/sendtohelix.proj /t:test - /v:diag /p:ArchGroup=${{ parameters.archGroup }} /p:ConfigurationGroup=${{ parameters.configuration }} /p:OSGroup=${{ parameters.targetOS }} From b668b575d1b64a45cfc29fc3a2b81c81ac90ad65 Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 3 Dec 2019 19:23:46 +0100 Subject: [PATCH 22/25] Address PR feedback --- eng/pipelines/coreclr/templates/build-test-job.yml | 7 +++++++ eng/pipelines/libraries/base-job.yml | 8 ++++---- eng/pipelines/libraries/build-job.yml | 9 ++++++++- eng/pipelines/libraries/build-test-job.yml | 2 +- eng/pipelines/runtime.yml | 13 +++++++++++++ 5 files changed, 33 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/coreclr/templates/build-test-job.yml b/eng/pipelines/coreclr/templates/build-test-job.yml index 74a39228bb9aab..5d473fd2755c38 100644 --- a/eng/pipelines/coreclr/templates/build-test-job.yml +++ b/eng/pipelines/coreclr/templates/build-test-job.yml @@ -5,7 +5,14 @@ parameters: osSubgroup: '' container: '' testGroup: '' + + # When set to a non-empty value (Debug / Release), it determines libraries + # build configuration to use for the tests. Setting this property implies + # a dependency of this job on the appropriate libraries build and is used + # to construct the name of the Azure artifact representing libraries build + # to use for building the tests. liveLibrariesBuildConfig: '' + displayNameArgs: '' condition: true stagedBuild: false diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index 5f14b09c9c1bcb..cfbb8037fbc3ec 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -30,7 +30,7 @@ jobs: helixRepo: dotnet/runtime pool: ${{ parameters.pool }} variables: - - buildScriptFileName: libraries + - _buildScriptFileName: libraries - _BuildConfig: ${{ parameters.buildConfig }} - _msbuildCommonParameters: '' @@ -38,7 +38,7 @@ jobs: - _runtimeOSArg: '' - _portableArg: '' - _finalFrameworkArg: -framework ${{ parameters.framework }} - - _buildScript: $(buildScriptFileName)$(scriptExt) + - _buildScript: $(_buildScriptFileName)$(scriptExt) - _warnAsErrorArg: '' - _testScopeArg: '' @@ -82,11 +82,11 @@ jobs: # Non-Windows variables - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - _buildScript: ./$(buildScriptFileName)$(scriptExt) + - _buildScript: ./$(_buildScriptFileName)$(scriptExt) - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - _stripSymbolsArg: -stripSymbols - - _buildArguments: -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) $(_stripSymbolsArg) $(_testScopeArg) $(_warnAsErrorArg) $(_runtimeOSArg) $(_portableArg) $(_msbuildCommonParameters) + - _buildArguments: -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) $(_stripSymbolsArg) $(_testScopeArg) $(_warnAsErrorArg) $(_runtimeOSArg) $(_portableArg) $(_msbuildCommonParameters) $(_coreClrOverridePathArg) - ${{ parameters.variables }} dependsOn: diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index 3947baec048c52..9e4bf1cc4d1e58 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -5,7 +5,14 @@ parameters: archType: '' framework: netcoreapp isOfficialBuild: false + + # When set to a non-empty value (Debug / Release), it determines CoreCLR + # build configuration to use for building libraries and tests. Setting this + # property implies a dependency of this job on the appropriate CoreCLR build + # and is used to construct the name of the Azure artifact representing + # CoreCLR build to use for building the libraries and library tests. liveCoreClrBuildConfig: '' + timeoutInMinutes: 150 preBuildSteps: [] container: '' @@ -50,7 +57,7 @@ jobs: - ${{ parameters.variables }} steps: - - script: $(_buildScript) -restore $(_buildArguments) $(_skipTestRestoreArg) $(_coreClrOverridePathArg) + - script: $(_buildScript) -restore $(_buildArguments) $(_skipTestRestoreArg) displayName: Restore - ${{ if eq(parameters.isOfficialBuild, true) }}: diff --git a/eng/pipelines/libraries/build-test-job.yml b/eng/pipelines/libraries/build-test-job.yml index 0857e2f300686e..c6da821670b237 100644 --- a/eng/pipelines/libraries/build-test-job.yml +++ b/eng/pipelines/libraries/build-test-job.yml @@ -49,7 +49,7 @@ jobs: unpackFolder: $(Build.SourcesDirectory)/artifacts cleanUnpackFolder: false - - script: $(_buildScript) -restore $(_buildArguments) $(_skipTestRestoreArg) $(_coreClrOverridePathArg) + - script: $(_buildScript) -restore $(_buildArguments) $(_skipTestRestoreArg) displayName: Restore - script: $(_buildScript) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 13f61f7d41b4e9..40ece6660ba5f6 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -22,8 +22,10 @@ jobs: platforms: - OSX_x64 - Linux_x64 + - Linux_arm - Windows_NT_x86 - Windows_NT_x64 + - Windows_NT_arm - Windows_NT_arm64 jobParameters: testGroup: innerloop @@ -38,8 +40,10 @@ jobs: platforms: - OSX_x64 - Linux_x64 + - Linux_arm - Windows_NT_x86 - Windows_NT_x64 + - Windows_NT_arm - Windows_NT_arm64 jobParameters: isOfficialBuild: false @@ -56,8 +60,10 @@ jobs: platforms: - OSX_x64 - Linux_x64 + - Linux_arm - Windows_NT_x86 - Windows_NT_x64 + - Windows_NT_arm - Windows_NT_arm64 jobParameters: testGroup: innerloop @@ -73,8 +79,10 @@ jobs: platforms: - OSX_x64 - Linux_x64 + - Linux_arm - Windows_NT_x86 - Windows_NT_x64 + - Windows_NT_arm - Windows_NT_arm64 helixQueueGroup: pr helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml @@ -92,8 +100,10 @@ jobs: platforms: - OSX_x64 - Linux_x64 + - Linux_arm - Windows_NT_x86 - Windows_NT_x64 + - Windows_NT_arm - Windows_NT_arm64 jobParameters: isOfficialBuild: false @@ -110,8 +120,11 @@ jobs: platforms: - OSX_x64 - Linux_x64 + - Linux_arm - Windows_NT_x86 - Windows_NT_x64 + - Windows_NT_arm + - Windows_NT_arm64 helixQueueGroup: pr helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml jobParameters: From 7674b40047761cdac0ed46a7caca369f8ba63c83 Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 3 Dec 2019 20:39:34 +0100 Subject: [PATCH 23/25] Latest bits of PR feedback and disabling test runs for now --- eng/pipelines/coreclr/templates/build-test-job.yml | 2 +- eng/pipelines/coreclr/templates/run-test-job.yml | 1 + eng/pipelines/runtime.yml | 10 ++++++---- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/coreclr/templates/build-test-job.yml b/eng/pipelines/coreclr/templates/build-test-job.yml index 5d473fd2755c38..15f8668b8e9aa2 100644 --- a/eng/pipelines/coreclr/templates/build-test-job.yml +++ b/eng/pipelines/coreclr/templates/build-test-job.yml @@ -96,7 +96,7 @@ jobs: cleanUnpackFolder: false artifactFileName: '$(librariesArtifactName)$(archiveExtension)' artifactName: '$(librariesArtifactName)' - displayName: 'live-built CoreFX' + displayName: 'live-built libraries' # Download product binaries directory diff --git a/eng/pipelines/coreclr/templates/run-test-job.yml b/eng/pipelines/coreclr/templates/run-test-job.yml index f2e8da1eb75007..bf854e1af11d11 100644 --- a/eng/pipelines/coreclr/templates/run-test-job.yml +++ b/eng/pipelines/coreclr/templates/run-test-job.yml @@ -14,6 +14,7 @@ parameters: helixQueues: '' # If true, run the corefx tests instead of the coreclr ones corefxTests: false + condition: true stagedBuild: false displayNameArgs: '' runInUnloadableContext: false diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 40ece6660ba5f6..1c4c1c7ae82120 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -31,7 +31,7 @@ jobs: testGroup: innerloop # -# Build CoreFX using live CoreLib from CoreCLR +# Build libraries using live CoreLib from CoreCLR # - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -51,7 +51,7 @@ jobs: liveCoreClrBuildConfig: checked # -# Test builds using live CoreFX +# Test builds using live libraries build # - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -89,9 +89,10 @@ jobs: jobParameters: testGroup: innerloop liveLibrariesBuildConfig: Release + condition: false # -# CoreFX Test Build +# Libraries Test Build # - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -111,7 +112,7 @@ jobs: liveCoreClrBuildConfig: checked # -# CoreFX Test Execution +# Libraries Test Execution # - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -134,3 +135,4 @@ jobs: testScope: innerloop liveCoreClrBuildConfig: checked dependsOnBuildConfiguration: Release + condition: false From 7402ce7c72f79b9ad037cf6c9d769eec0c7e1d76 Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 3 Dec 2019 20:50:53 +0100 Subject: [PATCH 24/25] Stop passing testScope: empty to libraries build --- eng/pipelines/runtime.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 1c4c1c7ae82120..f1cdbe016181d3 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -47,7 +47,6 @@ jobs: - Windows_NT_arm64 jobParameters: isOfficialBuild: false - testScope: empty liveCoreClrBuildConfig: checked # From 28a52c4a614e67c4157402a6a7a89306d9f2b845 Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 3 Dec 2019 21:52:36 +0100 Subject: [PATCH 25/25] Sigh - one last bit, propagation of condition wasn't working properly --- eng/pipelines/libraries/base-job.yml | 2 ++ eng/pipelines/libraries/build-test-job.yml | 2 ++ eng/pipelines/libraries/run-test-job.yml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index cfbb8037fbc3ec..cceb842179cb12 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -7,6 +7,7 @@ parameters: isOfficialBuild: false liveCoreClrBuildConfig: '' timeoutInMinutes: 150 + condition: true container: '' steps: [] dependsOn: [] @@ -27,6 +28,7 @@ jobs: enableTelemetry: ${{ parameters.isOfficialBuild }} # TODO: figure out if it's needed container: ${{ parameters.container }} + condition: ${{ parameters.condition }} helixRepo: dotnet/runtime pool: ${{ parameters.pool }} variables: diff --git a/eng/pipelines/libraries/build-test-job.yml b/eng/pipelines/libraries/build-test-job.yml index c6da821670b237..f7a91d45b97a06 100644 --- a/eng/pipelines/libraries/build-test-job.yml +++ b/eng/pipelines/libraries/build-test-job.yml @@ -12,6 +12,7 @@ parameters: pool: '' testScope: '' variables: {} + condition: true runTests: false jobs: @@ -23,6 +24,7 @@ jobs: archType: ${{ parameters.archType }} framework: ${{ parameters.framework }} isOfficialBuild: ${{ parameters.isOfficialBuild }} + condition: ${{ parameters.condition }} liveCoreClrBuildConfig: ${{ parameters.liveCoreClrBuildConfig }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} container: ${{ parameters.container }} diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index 06b2a7b47254ba..b662732c9d906e 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -11,6 +11,7 @@ parameters: testScope: '' helixQueues: [] dependsOnBuildConfiguration: Debug + condition: true variables: {} jobs: @@ -25,6 +26,7 @@ jobs: liveCoreClrBuildConfig: ${{ parameters.liveCoreClrBuildConfig }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} container: '' # we just send to helix, no need to use a container. + condition: ${{ parameters.condition }} pool: ${{ parameters.pool }} name: test_run displayName: 'Test Run'