From b0bc96291f982655639df60e547dbf5cd1ba3733 Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Fri, 15 Nov 2019 15:29:12 -0800 Subject: [PATCH 1/3] Initial work to make corefx templates re-usable for live live --- eng/pipelines/libraries/build.yml | 78 +++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 eng/pipelines/libraries/build.yml diff --git a/eng/pipelines/libraries/build.yml b/eng/pipelines/libraries/build.yml new file mode 100644 index 00000000000000..3ee51dbc2f3aa6 --- /dev/null +++ b/eng/pipelines/libraries/build.yml @@ -0,0 +1,78 @@ +parameters: + buildConfig: '' + osGroup: '' + archType: '' + osSubgroup: '' + framework: netcoreapp + isOfficialBuild: false + timeoutInMinutes: 150 + preBuildSteps: [] + +jobs: + - template: /eng/common/templates/job/job.yml + parameters: + variables: + - _msbuildCommonParameters: '' + - _archiveTestsParameter: '' + - _finalFrameworkArg: -framework ${{ parameters.framework }} + + - ${{ if eq(parameters.framework, 'allConfigurations' ) }}: + - _finalFrameworkArg: -allConfigurations + + - ${{ if eq(parameters.isOfficialBuild, 'true') }}: + - _msbuildCommonParameters: /p:OfficialBuildId=$(Build.BuildNumber) + + - _args: -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) + - _commonArguments: $(_args) + + # Windows variables + - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: + - _buildScript: $(buildScriptFileName).cmd + - _msbuildCommand: powershell -ExecutionPolicy ByPass -NoProfile eng\common\msbuild.ps1 -warnaserror:0 -ci + + # Non-Windows variables + - ${{ if ne(parameters.targetOS, 'Windows_NT') }}: + - _buildScript: ./$(buildScriptFileName).sh + - _msbuildCommand: ./eng/common/msbuild.sh --warnaserror false --ci + - ${{ if eq(parameters.isOfficialBuild, 'true') }}: + - _commonArguments: $(_args) -stripSymbols + + enableTelemetry: ${{ parameters.isOfficialBuild }} # TODO: figure out if it's needed + helixRepo: dotnet/runtime + + $ {{ if eq(parameters.framework, 'netcoreapp') }}: + name: ${{ format('build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + ${{ if ne(parameters.framework, 'netcoreapp') }}: + name: ${{ format('{0} {1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + + workspace: + clean: all + + enablePublishBuildArtifacts: true + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + steps: + - ${{ if ne(parameters.preBuildSteps[0], '') }}: + - ${{ parameters.preBuildSteps }} + + - script: $(_buildScript) -restore $(_commonArguments) + displayName: Restore + + - ${{ if eq(parameters.isOfficialBuild, 'true') }}: + - task: DotNetCoreCLI@2 + displayName: Restore internal tools + condition: and(succeeded(), ne(variables['_skipRestoreInternalTools'], 'true')) + inputs: + command: restore + feedsToUse: config + projects: 'eng/common/internal/Tools.csproj' + nugetConfigPath: 'eng/internal/NuGet.config' + restoreDirectory: '$(Build.SourcesDirectory)\.packages' + verbosityRestore: 'normal' + externalFeedCredentials: 'dotnet-core-internal-tooling' + + - script: $(_buildScript) + -build + $(_commonArguments) + $(_msbuildCommonParameters) + displayName: Build Sources \ No newline at end of file From 6b1ee48023c2eeea9d9fac73ae30b47d6409f75c Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Fri, 15 Nov 2019 16:48:02 -0800 Subject: [PATCH 2/3] Support building and running tests --- eng/pipelines/libraries/build.yml | 68 +++++++++++++++++++++++++------ eng/pipelines/libraries/helix.yml | 4 -- src/libraries/build.proj | 2 +- 3 files changed, 57 insertions(+), 17 deletions(-) diff --git a/eng/pipelines/libraries/build.yml b/eng/pipelines/libraries/build.yml index 3ee51dbc2f3aa6..b790a245046a28 100644 --- a/eng/pipelines/libraries/build.yml +++ b/eng/pipelines/libraries/build.yml @@ -7,14 +7,24 @@ parameters: isOfficialBuild: false timeoutInMinutes: 150 preBuildSteps: [] + testScope: '' + runTests: true + publishTestArtifacts: false jobs: - template: /eng/common/templates/job/job.yml parameters: variables: - _msbuildCommonParameters: '' - - _archiveTestsParameter: '' - _finalFrameworkArg: -framework ${{ parameters.framework }} + - _skipTestRestoreArg: '' + - _testScopeArg: '' + - _actionParameters: -build + + - ${{ if ne(parameters.testScope, '') }}: + - _testScopeArg: -testscope ${{ parameters.testScope }} + - _skipTestRestoreArg: /p:SkipTestRestore=true + - _actionParameters: -buildtests - ${{ if eq(parameters.framework, 'allConfigurations' ) }}: - _finalFrameworkArg: -allConfigurations @@ -22,7 +32,7 @@ jobs: - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - _msbuildCommonParameters: /p:OfficialBuildId=$(Build.BuildNumber) - - _args: -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) + - _args: -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) $(_testScopeArg) $(_skipTestRestoreArg) - _commonArguments: $(_args) # Windows variables @@ -36,18 +46,37 @@ jobs: - _msbuildCommand: ./eng/common/msbuild.sh --warnaserror false --ci - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - _commonArguments: $(_args) -stripSymbols - + + - ${{ if or(eq(parameters.runTests, 'true'), eq(parameters.publishTestArtifacts, 'true')) }}: + - ${{ if ne(parameters.framework, 'allConfigurations') }}: + - _commonArguments: $(_args) /p:ArchiveTests=Tests + - ${{ if eq(parameters.framework, 'allConfigurations') }}: + - _commonArguments: $(_args) /p:ArchiveTests=Packages + enableTelemetry: ${{ parameters.isOfficialBuild }} # TODO: figure out if it's needed helixRepo: dotnet/runtime - $ {{ if eq(parameters.framework, 'netcoreapp') }}: - name: ${{ format('build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - ${{ if ne(parameters.framework, 'netcoreapp') }}: - name: ${{ format('{0} {1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + ${{ if eq(parameters.testScope, '') }}: + name: ${{ format('libraries_build_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + $ {{ if eq(parameters.framework, 'netcoreapp') }}: + displayName: ${{ format('{0}{1} Build {2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + ${{ if ne(parameters.framework, 'netcoreapp') }}: + displayName: ${{ format('{0} Build {1} {2}_{3}', parameters.osGroup, parameters.framework, parameters.archType, parameters.buildConfig) }} + + ${{ if ne(parameters.testScope, '') }}: + name: ${{ format('libraries_test_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + $ {{ if eq(parameters.framework, 'netcoreapp') }}: + displayName: ${{ format('{0}{1} Test {2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + ${{ if ne(parameters.framework, 'netcoreapp') }}: + displayName: ${{ format('{0} Test {1} {2}_{3}', parameters.osGroup, parameters.framework, parameters.archType, parameters.buildConfig) }} + + ${{ if ne(parameters.testScope, '') }}: + dependencies: + - ${{ format('libraries_build_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} workspace: clean: all - + enablePublishBuildArtifacts: true timeoutInMinutes: ${{ parameters.timeoutInMinutes }} @@ -57,7 +86,7 @@ jobs: - script: $(_buildScript) -restore $(_commonArguments) displayName: Restore - + - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - task: DotNetCoreCLI@2 displayName: Restore internal tools @@ -70,9 +99,24 @@ jobs: restoreDirectory: '$(Build.SourcesDirectory)\.packages' verbosityRestore: 'normal' externalFeedCredentials: 'dotnet-core-internal-tooling' - + - script: $(_buildScript) - -build + $(_actionParameters) $(_commonArguments) $(_msbuildCommonParameters) - displayName: Build Sources \ No newline at end of file + displayName: Build + + - ${{ if and(eq(parameters.runTests, 'true'), ne(parameters.isOfficialBuild, 'false')) }}: + - template: /eng/pipelines/libraries/helix.yml + parameters: + targetOS: ${{ parameters.targetOS }} + archGroup: ${{ parameters.archType }} + configuration: ${{ parameters.buildConfig }} + helixQueues: $(_helixQueues) + msbuildScript: $(_msbuildCommand) + framework: ${{ parameters.framework }} + testScope: ${{ parameters.testScope }} + creator: dotnet-bot + helixToken: '' + + # TODO: publish test artifacts when publishTestArtifacts == true. diff --git a/eng/pipelines/libraries/helix.yml b/eng/pipelines/libraries/helix.yml index 8b20f44624de5b..4f5f2d28af3c45 100644 --- a/eng/pipelines/libraries/helix.yml +++ b/eng/pipelines/libraries/helix.yml @@ -4,10 +4,8 @@ parameters: creator: '' framework: '' helixQueues: '' - helixToken: '' msbuildScript: '' targetOS: '' - officialBuildId: '' testScope: 'innerloop' # innerloop | outerloop | all condition: always() @@ -22,9 +20,7 @@ steps: /p:TargetGroup=${{ parameters.framework }} /p:HelixTargetQueues=${{ parameters.helixQueues }} /p:HelixBuild=$(Build.BuildNumber) - /p:HelixAccessToken=${{ parameters.helixToken }} /p:Creator=${{ parameters.creator }} - /p:OfficialBuildId=${{ parameters.officialBuildId }} /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog displayName: Send to Helix condition: and(succeeded(), ${{ parameters.condition }}) diff --git a/src/libraries/build.proj b/src/libraries/build.proj index 2d14f266047e6a..f132086061a01c 100644 --- a/src/libraries/build.proj +++ b/src/libraries/build.proj @@ -23,7 +23,7 @@ <_RestoreProjects Include="$(MSBuildThisFileDirectory)restore\dirs.proj" Build="true" /> - + <_RestoreProjects Include="$(MSBuildThisFileDirectory)tests.proj" Build="false" /> From 1e2d07df389248c769d91b638a47962bf8f33ffa Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Mon, 18 Nov 2019 14:19:01 -0800 Subject: [PATCH 3/3] First try to use platform-matrix --- eng/pipelines/libraries/.azure-ci.yml | 52 +++++++++++------------ eng/pipelines/libraries/build.yml | 59 ++++++++++++++++----------- eng/pipelines/libraries/variables.yml | 2 + 3 files changed, 62 insertions(+), 51 deletions(-) diff --git a/eng/pipelines/libraries/.azure-ci.yml b/eng/pipelines/libraries/.azure-ci.yml index 8a879d9fdf9d8a..209398d1afc1b1 100644 --- a/eng/pipelines/libraries/.azure-ci.yml +++ b/eng/pipelines/libraries/.azure-ci.yml @@ -82,39 +82,35 @@ variables: value: .NETCore stages: - # Windows legs - - template: ${{ variables['pipelinesPath'] }}/windows.yml - parameters: - isOfficialBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }} - fullMatrix: ${{ notIn(variables['Build.Reason'], 'PullRequest') }} - # Linux legs - - template: ${{ variables['pipelinesPath'] }}/linux.yml - parameters: - isOfficialBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }} - fullMatrix: ${{ notIn(variables['Build.Reason'], 'PullRequest') }} - - # MacOS legs - - template: ${{ variables['pipelinesPath'] }}/macos.yml - parameters: - isOfficialBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }} - fullMatrix: ${{ notIn(variables['Build.Reason'], 'PullRequest') }} - - # FreeBSD leg is only for official builds - # - template: ${{ variables['pipelinesPath'] }}/freebsd.yml - # parameters: - # isOfficialBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }} - # fullMatrix: ${{ notIn(variables['Build.Reason'], 'PullRequest') }} + - stage: Build + jobs: + - template: /eng/pipelines/coreclr/templates/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build.yml + buildConfig: Release + platforms: + - Windows_NT_x64 + jobParameters: + isOfficialBuild: ${{ variables['isOfficialBuild'] }} + + - template: /eng/pipelines/coreclr/templates/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build.yml + buildConfig: Release + platforms: + - Windows_NT_x64 + jobParameters: + isOfficialBuild: ${{ variables['isOfficialBuild'] }} + runTests: true + testScope: innerloop # Publish and validation steps. Only run in official builds - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }}: - - template: ${{ variables['pipelinesPath'] }}/pre-publish.yml + - ${{ if eq(variables['isOfficialBuild'], 'true') }}: + - template: /eng/pipelines/libraries/pre-publish.yml parameters: dependsOn: - - WindowsStage - - LinuxStage - - MacOSStage - # - FreeBSD + - Build - template: eng\common\templates\post-build\post-build.yml parameters: diff --git a/eng/pipelines/libraries/build.yml b/eng/pipelines/libraries/build.yml index b790a245046a28..301c75ec84cc0f 100644 --- a/eng/pipelines/libraries/build.yml +++ b/eng/pipelines/libraries/build.yml @@ -8,8 +8,12 @@ parameters: timeoutInMinutes: 150 preBuildSteps: [] testScope: '' - runTests: true + publishBinaryArtifacts: false + runTests: false publishTestArtifacts: false + helixQueues: [] + container: '' + crossrootfsDir: '' jobs: - template: /eng/common/templates/job/job.yml @@ -17,19 +21,22 @@ jobs: variables: - _msbuildCommonParameters: '' - _finalFrameworkArg: -framework ${{ parameters.framework }} - - _skipTestRestoreArg: '' + - _skipTestRestoreArg: /p:SkipTestRestore=true - _testScopeArg: '' - _actionParameters: -build + - ${{ if ne(parameters.crossrootfsDir, '') }}: + - ROOTFS_DIR: ${{ parameters.crossrootfsDir}} + - ${{ if ne(parameters.testScope, '') }}: - _testScopeArg: -testscope ${{ parameters.testScope }} - - _skipTestRestoreArg: /p:SkipTestRestore=true + - _skipTestRestoreArg: '' - _actionParameters: -buildtests - ${{ if eq(parameters.framework, 'allConfigurations' ) }}: - _finalFrameworkArg: -allConfigurations - - ${{ if eq(parameters.isOfficialBuild, 'true') }}: + - ${{ if eq(parameters.isOfficialBuild, true) }}: - _msbuildCommonParameters: /p:OfficialBuildId=$(Build.BuildNumber) - _args: -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) $(_testScopeArg) $(_skipTestRestoreArg) @@ -41,13 +48,13 @@ jobs: - _msbuildCommand: powershell -ExecutionPolicy ByPass -NoProfile eng\common\msbuild.ps1 -warnaserror:0 -ci # Non-Windows variables - - ${{ if ne(parameters.targetOS, 'Windows_NT') }}: + - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - _buildScript: ./$(buildScriptFileName).sh - _msbuildCommand: ./eng/common/msbuild.sh --warnaserror false --ci - - ${{ if eq(parameters.isOfficialBuild, 'true') }}: + - ${{ if eq(parameters.isOfficialBuild, true) }}: - _commonArguments: $(_args) -stripSymbols - - ${{ if or(eq(parameters.runTests, 'true'), eq(parameters.publishTestArtifacts, 'true')) }}: + - ${{ if or(eq(parameters.runTests, true), eq(parameters.publishTestArtifacts, true)) }}: - ${{ if ne(parameters.framework, 'allConfigurations') }}: - _commonArguments: $(_args) /p:ArchiveTests=Tests - ${{ if eq(parameters.framework, 'allConfigurations') }}: @@ -56,22 +63,28 @@ jobs: enableTelemetry: ${{ parameters.isOfficialBuild }} # TODO: figure out if it's needed helixRepo: dotnet/runtime + ${{ if ne(parameters.container, '') }}: + ${{ if eq(parameters.container.registry, 'mcr') }}: + container: ${{ format('{0}:{1}', 'mcr.microsoft.com/dotnet-buildtools/prereqs', parameters.container.image) }} + ${{ if ne(parameters.container.registry, 'mcr') }}: + container: ${{ format('{0}:{1}', parameters.container.registry, parameters.container.image) }} + ${{ if eq(parameters.testScope, '') }}: name: ${{ format('libraries_build_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - $ {{ if eq(parameters.framework, 'netcoreapp') }}: + ${{ if eq(parameters.framework, 'netcoreapp') }}: displayName: ${{ format('{0}{1} Build {2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} ${{ if ne(parameters.framework, 'netcoreapp') }}: displayName: ${{ format('{0} Build {1} {2}_{3}', parameters.osGroup, parameters.framework, parameters.archType, parameters.buildConfig) }} ${{ if ne(parameters.testScope, '') }}: name: ${{ format('libraries_test_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - $ {{ if eq(parameters.framework, 'netcoreapp') }}: + ${{ if eq(parameters.framework, 'netcoreapp') }}: displayName: ${{ format('{0}{1} Test {2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} ${{ if ne(parameters.framework, 'netcoreapp') }}: displayName: ${{ format('{0} Test {1} {2}_{3}', parameters.osGroup, parameters.framework, parameters.archType, parameters.buildConfig) }} ${{ if ne(parameters.testScope, '') }}: - dependencies: + dependsOn: - ${{ format('libraries_build_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} workspace: @@ -87,10 +100,10 @@ jobs: - script: $(_buildScript) -restore $(_commonArguments) displayName: Restore - - ${{ if eq(parameters.isOfficialBuild, 'true') }}: + - ${{ if eq(parameters.isOfficialBuild, true) }}: - task: DotNetCoreCLI@2 displayName: Restore internal tools - condition: and(succeeded(), ne(variables['_skipRestoreInternalTools'], 'true')) + condition: and(succeeded(), ne(variables['_skipRestoreInternalTools'], true)) inputs: command: restore feedsToUse: config @@ -106,17 +119,17 @@ jobs: $(_msbuildCommonParameters) displayName: Build - - ${{ if and(eq(parameters.runTests, 'true'), ne(parameters.isOfficialBuild, 'false')) }}: + - ${{ if eq(parameters.runTests, true) }}: - template: /eng/pipelines/libraries/helix.yml - parameters: - targetOS: ${{ parameters.targetOS }} - archGroup: ${{ parameters.archType }} - configuration: ${{ parameters.buildConfig }} - helixQueues: $(_helixQueues) - msbuildScript: $(_msbuildCommand) - framework: ${{ parameters.framework }} - testScope: ${{ parameters.testScope }} - creator: dotnet-bot - helixToken: '' + parameters: + targetOS: ${{ parameters.targetOS }} + archGroup: ${{ parameters.archType }} + configuration: ${{ parameters.buildConfig }} + helixQueues: ${{ join(';', parameters.helixQueues) }} + msbuildScript: $(_msbuildCommand) + framework: ${{ parameters.framework }} + testScope: ${{ parameters.testScope }} + creator: dotnet-bot + helixToken: '' # TODO: publish test artifacts when publishTestArtifacts == true. diff --git a/eng/pipelines/libraries/variables.yml b/eng/pipelines/libraries/variables.yml index 2e0cf9f6da83f0..f0d229fc74f102 100644 --- a/eng/pipelines/libraries/variables.yml +++ b/eng/pipelines/libraries/variables.yml @@ -5,3 +5,5 @@ variables: value: $(Build.SourcesDirectory)/src/libraries - name: pipelinesPath value: /eng/pipelines/libraries + - name: isOfficialBuild + value: ${{ and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }}