diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 87122cb7a73..ba2797c39a3 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -35,9 +35,9 @@ - + https://github.com/dotnet/arcade - 58be4cadf2cabd4b3b3186576557835a1987a711 + 9a5da1c8a3232c6304955c7a6ffe6109a1b13d9f diff --git a/eng/common/PublishBuildAssets.cmd b/eng/common/PublishBuildAssets.cmd deleted file mode 100644 index 3c6e4ff829d..00000000000 --- a/eng/common/PublishBuildAssets.cmd +++ /dev/null @@ -1,3 +0,0 @@ -@echo off -powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0sdk-task.ps1""" -msbuildEngine dotnet -restore -projects PublishBuildAssets.proj -ci %*" -exit /b %ErrorLevel% diff --git a/eng/common/PublishToPackageFeed.proj b/eng/common/PublishToPackageFeed.proj index 9db45389486..8149e3fb6a1 100644 --- a/eng/common/PublishToPackageFeed.proj +++ b/eng/common/PublishToPackageFeed.proj @@ -32,6 +32,9 @@ Build configuration: 'Debug' or 'Release' (short: -c)" Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" - Write-Host " -binaryLog Output binary log; specify name of Binary Log in the form .binlog (short: -bl)" + Write-Host " -binaryLog Output binary log (short: -bl)" Write-Host " -help Print help and exit" Write-Host "" @@ -74,14 +74,7 @@ function Build { $toolsetBuildProj = InitializeToolset InitializeCustomToolset - $bl = "" - # if flag is present - if ($null -ne $binaryLog) - { - # if value is set, then use it; otherwise default to Build.binlog - $binaryLogName = if ("" -eq $binaryLog) { "Build" } else { $binaryLog } - $bl = "/bl:" + (Join-Path $LogDir ($binaryLogName + ".binlog")) - } + $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "Build.binlog") } else { "" } if ($projects) { # Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons. @@ -107,19 +100,17 @@ function Build { /p:Sign=$sign ` /p:Publish=$publish ` /p:ContinuousIntegrationBuild=$ci ` - /p:AllowUnsafeBlocks=true ` @properties } try { - if ($help -or (($null -ne $properties) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) { + if ($help -or (($properties -ne $null) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) { Print-Usage exit 0 } if ($ci) { - # if binarylog value is given, do not overwrite it - $binaryLog = if ($null -eq $binaryLog) { "" } else { $binaryLog } + $binaryLog = $true $nodeReuse = $false } diff --git a/eng/common/internal/Directory.Build.props b/eng/common/internal/Directory.Build.props new file mode 100644 index 00000000000..e33179ef373 --- /dev/null +++ b/eng/common/internal/Directory.Build.props @@ -0,0 +1,4 @@ + + + + diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj new file mode 100644 index 00000000000..1a81ff906f6 --- /dev/null +++ b/eng/common/internal/Tools.csproj @@ -0,0 +1,23 @@ + + + + + net472 + false + + + + + + + + + + https://devdiv.pkgs.visualstudio.com/_packaging/8f470c7e-ac49-4afe-a6ee-cf784e438b93/nuget/v3/index.json; + https://devdiv.pkgs.visualstudio.com/_packaging/VS/nuget/v3/index.json; + + + + + + diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 9ba7530122f..d0eec5163ef 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -1,48 +1,47 @@ [CmdletBinding(PositionalBinding=$false)] Param( - [string] $projects = "", - [string][Alias('v')]$verbosity = "minimal", + [string] $configuration = "Debug", + [string] $task, + [string] $verbosity = "minimal", [string] $msbuildEngine = $null, - [bool] $warnAsError = $true, - [switch][Alias('bl')]$binaryLog, - [switch][Alias('r')]$restore, - [switch] $ci, + [switch] $restore, [switch] $prepareMachine, [switch] $help, [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties ) +$ci = $true +$binaryLog = $true +$warnAsError = $true + . $PSScriptRoot\tools.ps1 function Print-Usage() { - Write-Host "Common settings:" - Write-Host " -v[erbosity] Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]" - Write-Host " -[bl|binaryLog] Output binary log (short: -bl)" - Write-Host " -help Print help and exit" - Write-Host "" + Write-Host "Common settings:" + Write-Host " -task Name of Arcade task (name of a project in SdkTasks directory of the Arcade SDK package)" + Write-Host " -restore Restore dependencies" + Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]" + Write-Host " -help Print help and exit" + Write-Host "" - Write-Host "Advanced settings:" - Write-Host " -restore Restore dependencies (short: -r)" - Write-Host " -projects Semi-colon delimited list of sln/proj's from the Arcade sdk to build. Globbing is supported (*.sln)" - Write-Host " -ci Set when running on CI server" - Write-Host " -prepareMachine Prepare machine for CI run" - Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." - Write-Host "" - Write-Host "Command line arguments not listed above are passed thru to msbuild." - Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)." + Write-Host "Advanced settings:" + Write-Host " -prepareMachine Prepare machine for CI run" + Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." + Write-Host "" + Write-Host "Command line arguments not listed above are passed thru to msbuild." } -function Build { - $toolsetBuildProj = InitializeToolset +function Build([string]$target) { + $logSuffix = if ($target -eq "Execute") { "" } else { ".$target" } + $log = Join-Path $LogDir "$task$logSuffix.binlog" + $outputPath = Join-Path $ToolsetDir "$task\\" - $toolsetBuildProj = Join-Path (Split-Path $toolsetBuildProj -Parent) "SdkTasks\SdkTask.proj" - $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "SdkTask.binlog") } else { "" } - MSBuild $toolsetBuildProj ` - $bl ` - /p:Projects=$projects ` - /p:Restore=$restore ` + MSBuild $taskProject ` + /bl:$log ` + /t:$target ` + /p:Configuration=$configuration ` /p:RepoRoot=$RepoRoot ` - /p:ContinuousIntegrationBuild=$ci ` + /p:BaseIntermediateOutputPath=$outputPath ` @properties } @@ -52,17 +51,23 @@ try { exit 0 } - if ($projects -eq "") { - Write-Error "Missing required parameter '-projects '" + if ($task -eq "") { + Write-Host "Missing required parameter '-task '" -ForegroundColor Red Print-Usage ExitWithExitCode 1 } - if ($ci) { - $binaryLog = $true + $taskProject = GetSdkTaskProject $task + if (!(Test-Path $taskProject)) { + Write-Host "Unknown task: $task" -ForegroundColor Red + ExitWithExitCode 1 + } + + if ($restore) { + Build "Restore" } - Build + Build "Execute" } catch { Write-Host $_ diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 5e293db35d7..98ca94c326c 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -41,18 +41,11 @@ parameters: enablePublishTestResults: false # Optional: enable sending telemetry - # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix - # _HelixBuildConfig - differentiate between Debug, Release, other - # _HelixType - Example: build/product/ - # _HelixSource - Example: official/dotnet/arcade/$(Build.SourceBranch) enableTelemetry: false - # Optional: If specified, then automatically derive "_HelixSource" variable for telemetry + # Optional: define the helix repo for telemeetry (example: 'dotnet/arcade') helixRepo: '' - # Optional: Define the type for helix telemetry (must end in '/') - helixType: build/product/ - # Required: name of the job name: '' @@ -115,29 +108,20 @@ jobs: - name: ${{ pair.key }} value: ${{ pair.value }} - # Add additional variables - - ${{ if and(ne(parameters.helixRepo, ''), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }}: - - name: _HelixSource - value: official/${{ parameters.helixRepo }}/$(Build.SourceBranch) - - ${{ if and(ne(parameters.helixRepo, ''), or(ne(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest'))) }}: - - name: _HelixSource - value: pr/${{ parameters.helixRepo }}/$(Build.SourceBranch) - - name: _HelixType - value: ${{ parameters.helixType }} - - name: _HelixBuildConfig - value: $(_BuildConfig) - ${{ if ne(parameters.workspace, '') }}: workspace: ${{ parameters.workspace }} steps: - ${{ if eq(parameters.enableTelemetry, 'true') }}: - - template: /eng/common/templates/steps/telemetry-start.yml - parameters: - buildConfig: $(_HelixBuildConfig) - helixSource: $(_HelixSource) - helixType: $(_HelixType) + # Telemetry tasks are built from https://github.com/dotnet/arcade-extensions + - task: sendStartTelemetry@0 + displayName: 'Send Helix Start Telemetry' + inputs: + helixRepo: ${{ parameters.helixRepo }} + buildConfig: $(_BuildConfig) runAsPublic: ${{ parameters.runAsPublic }} + continueOnError: ${{ parameters.continueOnError }} + condition: always() - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: @@ -165,11 +149,15 @@ jobs: TeamName: $(_TeamName) - ${{ if eq(parameters.enableTelemetry, 'true') }}: - - template: /eng/common/templates/steps/telemetry-end.yml + # Telemetry tasks are built from https://github.com/dotnet/arcade-extensions + - task: sendEndTelemetry@0 + displayName: 'Send Helix End Telemetry' + continueOnError: ${{ parameters.continueOnError }} + condition: always() - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - task: PublishBuildArtifacts@1 - displayName: Publish Logs to VSTS + displayName: Publish Logs inputs: PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' PublishLocation: Container diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index c094658fefc..d6d8697cbde 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -44,12 +44,15 @@ jobs: downloadPath: '$(Build.StagingDirectory)/Download' condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - - script: eng\common\publishbuildassets.cmd - /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:BuildAssetRegistryToken=$(MaestroAccessToken) - /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com - /p:Configuration=$(_BuildConfig) + - task: PowerShell@2 displayName: Publish Build Assets + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' + /p:BuildAssetRegistryToken=$(MaestroAccessToken) + /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com + /p:Configuration=$(_BuildConfig) condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index c7226b12ed2..c1a5b4849ac 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -18,21 +18,15 @@ parameters: enablePublishTestResults: false # Optional: enable sending telemetry - # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix - # _HelixBuildConfig - differentiate between Debug, Release, other - # _HelixType - Example: build/product/ - # _HelixSource - Example: official/dotnet/arcade/$(Build.SourceBranch) + # if enabled then the 'helixRepo' parameter should also be specified enableTelemetry: false # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job jobs: [] - # Optional: If specified, then automatically derive "_HelixSource" variable for telemetry + # Optional: define the helix repo for telemetry (example: 'dotnet/arcade') helixRepo: '' - # Optional: Define the type for helix telemetry (must end in '/') - helixType: build/product/ - # Optional: Override automatically derived dependsOn value for "publish build assets" job publishBuildAssetsDependsOn: '' diff --git a/eng/common/templates/phases/publish-build-assets.yml b/eng/common/templates/phases/publish-build-assets.yml index 85fabe62b98..211967debab 100644 --- a/eng/common/templates/phases/publish-build-assets.yml +++ b/eng/common/templates/phases/publish-build-assets.yml @@ -28,12 +28,15 @@ phases: SecretsFilter: 'MaestroAccessToken' condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - - script: eng\common\publishbuildassets.cmd - /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:BuildAssetRegistryToken=$(MaestroAccessToken) - /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com - /p:Configuration=$(_BuildConfig) + - task: PowerShell@2 displayName: Publish Build Assets + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' + /p:BuildAssetRegistryToken=$(MaestroAccessToken) + /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com + /p:Configuration=$(_BuildConfig) condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - task: PublishBuildArtifacts@1 diff --git a/eng/common/templates/steps/telemetry-start.yml b/eng/common/templates/steps/telemetry-start.yml index f6a521be21e..32c01ef0b55 100644 --- a/eng/common/templates/steps/telemetry-start.yml +++ b/eng/common/templates/steps/telemetry-start.yml @@ -201,6 +201,7 @@ steps: MaxRetries: ${{ parameters.maxRetries }} RetryDelay: ${{ parameters.retryDelay }} condition: and(always(), eq(variables['Agent.Os'], 'Windows_NT')) + displayName: Send Windows Job Start Telemetry - powershell: | # Basic retry loop to harden against server flakiness $retryCount = 0 diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index d396dfac7a6..d1ed296dd6b 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -7,9 +7,9 @@ # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. [string]$configuration = if (Test-Path variable:configuration) { $configuration } else { "Debug" } -# Binary log name; give to output binary log from msbuild. Note that emitting binary log slows down the build. +# Set to true to output binary log from msbuild. Note that emitting binary log slows down the build. # Binary log must be enabled on CI. -[string]$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { if ($ci) { "" } else { $null } } +[bool]$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci } # Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes). [bool]$prepareMachine = if (Test-Path variable:prepareMachine) { $prepareMachine } else { $false } @@ -161,9 +161,10 @@ function GetDotNetInstallScript([string] $dotnetRoot) { return $installScript } -function InstallDotNetSdk([string] $dotnetRoot, [string] $version) { +function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = "") { $installScript = GetDotNetInstallScript $dotnetRoot - & $installScript -Version $version -InstallDir $dotnetRoot + $archArg = if ($architecture) { $architecture } else { "" } + & $installScript -Version $version -InstallDir $dotnetRoot -Architecture $archArg if ($lastExitCode -ne 0) { Write-Host "Failed to install dotnet cli (exit code '$lastExitCode')." -ForegroundColor Red ExitWithExitCode $lastExitCode @@ -210,7 +211,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0] InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion - } elseif ($install) { + } else { if (Get-Member -InputObject $GlobalJson.tools -Name "xcopy-msbuild") { $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild' @@ -220,9 +221,10 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $xcopyMSBuildVersion = "$vsMajorVersion.$($vsMinVersion.Minor).0-alpha" } - $vsInstallDir = InstallXCopyMSBuild $xcopyMSBuildVersion - } else { - throw "Unable to find Visual Studio that has required version and components installed" + $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install + if ($vsInstallDir -eq $null) { + throw "Unable to find Visual Studio that has required version and components installed" + } } $msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" } @@ -240,12 +242,20 @@ function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [str } } -function InstallXCopyMSBuild([string] $packageVersion) { +function InstallXCopyMSBuild([string]$packageVersion) { + return InitializeXCopyMSBuild $packageVersion -install $true +} + +function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { $packageName = "RoslynTools.MSBuild" $packageDir = Join-Path $ToolsDir "msbuild\$packageVersion" $packagePath = Join-Path $packageDir "$packageName.$packageVersion.nupkg" if (!(Test-Path $packageDir)) { + if (!$install) { + return $null + } + Create-Directory $packageDir Write-Host "Downloading $packageName $packageVersion" Invoke-WebRequest "https://dotnet.myget.org/F/roslyn-tools/api/v2/package/$packageName/$packageVersion/" -OutFile $packagePath @@ -406,15 +416,7 @@ function InitializeToolset() { $buildTool = InitializeBuildTool $proj = Join-Path $ToolsetDir "restore.proj" - - $bl = "" - # if flag is present - if ($null -ne $binaryLog) - { - # if value is set, then use it; otherwise default to ToolsetRestore.binlog - $binaryLogName = if ("" -eq $binaryLog) { "ToolsetRestore" } else { $binaryLog } - $bl = "/bl:" + (Join-Path $LogDir ($binaryLogName + ".binlog")) - } + $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "ToolsetRestore.binlog") } else { "" } '' | Set-Content $proj MSBuild $proj $bl /t:__WriteToolsetLocation /noconsolelogger /p:__ToolsetLocationOutputFile=$toolsetLocationFile @@ -448,8 +450,8 @@ function Stop-Processes() { # function MSBuild() { if ($ci) { - if ($null -eq $binaryLog) { - throw "Binary log must not be null in CI build." + if (!$binaryLog) { + throw "Binary log must be enabled in CI build." } if ($nodeReuse) { diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 8a253bef513..115a56a827d 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -153,7 +153,12 @@ function InstallDotNetSdk { GetDotNetInstallScript "$root" local install_script=$_GetDotNetInstallScript - bash "$install_script" --version $version --install-dir "$root" || { + local arch_arg="" + if [[ $# == 3 ]]; then + arch_arg="--architecture $3" + fi + + bash "$install_script" --version $version --install-dir "$root" $arch_arg || { local exit_code=$? echo "Failed to install dotnet SDK (exit code '$exit_code')." >&2 ExitWithExitCode $exit_code diff --git a/global.json b/global.json index bab4ae251d1..40242c0036b 100644 --- a/global.json +++ b/global.json @@ -9,6 +9,6 @@ "version": "3.0.100-preview-010024" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19103.1" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19112.3" } }