[master] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -3,13 +3,13 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20207.4">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20214.6">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>ac9c5c72b6fbb0e025006bc930b45db27e28c5ae</Sha>
+      <Sha>5f64bc0c3daca8298b9bda02ee39da6e2eed8b03</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="5.0.0-beta.20207.4">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="5.0.0-beta.20214.6">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>ac9c5c72b6fbb0e025006bc930b45db27e28c5ae</Sha>
+      <Sha>5f64bc0c3daca8298b9bda02ee39da6e2eed8b03</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/common/build.ps1

@@ -20,6 +20,7 @@ Param(
   [switch] $publish,
   [switch] $clean,
   [switch][Alias('bl')]$binaryLog,
+  [switch] $manifest,
   [switch] $ci,
   [switch] $prepareMachine,
   [switch] $help,
@@ -61,6 +62,7 @@ function Print-Usage() {
   Write-Host "  -prepareMachine         Prepare machine for CI run, clean up processes after build"
   Write-Host "  -warnAsError <value>    Sets warnaserror msbuild parameter ('true' or 'false')"
   Write-Host "  -msbuildEngine <value>  Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
+  Write-Host "  -manifest               Generates a build manifest, including signing metadata.  Requires 'publish' also be specified."
   Write-Host ""
 
   Write-Host "Command line arguments not listed above are passed thru to msbuild."
@@ -100,6 +102,13 @@ function Build {
     $properties = $msbuildArgs
   }
 
+  if ($manifest) {
+    if (-not $publish) {
+      Write-Host "'-publish' is required if '-manifest' is specified."
+      ExitWithExitCode 1
+    }
+  }
+
   MSBuild $toolsetBuildProj `
     $bl `
     $platformArg `
@@ -116,6 +125,7 @@ function Build {
     /p:PerformanceTest=$performanceTest `
     /p:Sign=$sign `
     /p:Publish=$publish `
+    /p:GenerateBuildManifest=$manifest `
     @properties
 }
 

eng/common/build.sh

@@ -35,6 +35,7 @@ usage()
   echo "  --prepareMachine         Prepare machine for CI run, clean up processes after build"
   echo "  --nodeReuse <value>      Sets nodereuse msbuild parameter ('true' or 'false')"
   echo "  --warnAsError <value>    Sets warnaserror msbuild parameter ('true' or 'false')"
+  echo "  --manifest               Generates a build manifest, including signing metadata.  Requires 'publish' also be specified."
   echo ""
   echo "Command line arguments not listed above are passed thru to msbuild."
   echo "Arguments can also be passed in with a single hyphen."
@@ -64,6 +65,7 @@ sign=false
 public=false
 ci=false
 clean=false
+manifest=false
 
 warn_as_error=true
 node_reuse=true
@@ -146,6 +148,9 @@ while [[ $# > 0 ]]; do
       node_reuse=$2
       shift
       ;;
+    -manifest)
+      manifest=true
+      ;;
     *)
       properties="$properties $1"
       ;;
@@ -183,6 +188,13 @@ function Build {
     bl="/bl:\"$log_dir/Build.binlog\""
   fi
 
+  if [[ "$manifest" == true ]]; then
+    if [[ ! "$publish" == true ]]; then
+      echo "'--publish' is required if '-manifest' is specified."
+      exit 0
+    fi
+  fi
+
   MSBuild $_InitializeToolset \
     $bl \
     /p:Configuration=$configuration \
@@ -196,6 +208,7 @@ function Build {
     /p:PerformanceTest=$performance_test \
     /p:Sign=$sign \
     /p:Publish=$publish \
+    /p:GenerateBuildManifest=$manifest \
     $properties
 
   ExitWithExitCode 0

eng/common/post-build/symbols-validation.ps1

@@ -1,7 +1,8 @@
 param(
   [Parameter(Mandatory=$true)][string] $InputPath,              # Full path to directory where NuGet packages to be checked are stored
   [Parameter(Mandatory=$true)][string] $ExtractPath,            # Full path to directory where the packages will be extracted during validation
-  [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion     # Version of dotnet symbol to use
+  [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion,    # Version of dotnet symbol to use
+  [Parameter(Mandatory=$false)][switch] $ContinueOnError        # If we should keep checking symbols after an error
 )
 
 function FirstMatchingSymbolDescriptionOrDefault {
@@ -125,6 +126,8 @@ function CheckSymbolsAvailable {
     Remove-Item $ExtractPath -Force  -Recurse -ErrorAction SilentlyContinue
   }
 
+  $TotalFailures = 0
+
   Get-ChildItem "$InputPath\*.nupkg" |
     ForEach-Object {
       $FileName = $_.Name
@@ -148,11 +151,22 @@ function CheckSymbolsAvailable {
 
       if ($Status -ne 0) {
         Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $Status modules in the package $FileName"
-        ExitWithExitCode $exitCode
+
+        if ($ContinueOnError) {
+          $TotalFailures++
+        }
+        else {
+          ExitWithExitCode 1
+        }
       }
 
       Write-Host
     }
+
+  if ($TotalFailures -ne 0) {
+    Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures packages"
+    ExitWithExitCode 1
+  }
 }
 
 function InstallDotnetSymbol {

eng/common/sdk-task.ps1

@@ -57,6 +57,18 @@ try {
     ExitWithExitCode 1
   }
 
+  if( $msbuildEngine -eq "vs") {
+    # Ensure desktop MSBuild is available for sdk tasks.
+    if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "vs" )) {
+      $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.4`" }") -MemberType NoteProperty
+    }
+    if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
+      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.4.0-alpha" -MemberType NoteProperty
+    }
+
+    InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true
+  }
+
   $taskProject = GetSdkTaskProject $task
   if (!(Test-Path $taskProject)) {
     Write-PipelineTelemetryError -Category 'Build' -Message "Unknown task: $task" -ForegroundColor Red

eng/common/sdl/packages.config

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="Microsoft.Guardian.Cli" version="0.7.2"/>
+  <package id="Microsoft.Guardian.Cli.win10-x64" version="0.20.1"/>
 </packages>

eng/common/templates/job/execute-sdl.yml

@@ -6,6 +6,7 @@ parameters:
   # 'continueOnError', the parameter value is not correctly picked up.
   # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter
   sdlContinueOnError: false                                    # optional: determines whether to continue the build if the step errors;
+  downloadArtifacts: true                                      # optional: determines if the artifacts should be dowloaded
   dependsOn: ''                                                # Optional: dependencies of the job
   artifactNames: ''                                            # Optional: patterns supplied to DownloadBuildArtifacts
                                                                # Usage:
@@ -31,8 +32,20 @@ jobs:
   steps:
   - checkout: self
     clean: true
-  - ${{ if ne(parameters.artifactNames, '') }}:
-    - ${{ each artifactName in parameters.artifactNames }}:
+  - ${{ if ne(parameters.downloadArtifacts, 'false')}}:
+    - ${{ if ne(parameters.artifactNames, '') }}:
+      - ${{ each artifactName in parameters.artifactNames }}:
+        - task: DownloadBuildArtifacts@0
+          displayName: Download Build Artifacts
+          inputs:
+            buildType: specific
+            buildVersionToDownload: specific
+            project: $(AzDOProjectName)
+            pipeline: $(AzDOPipelineId)
+            buildId: $(AzDOBuildId)
+            artifactName: ${{ artifactName }}
+            downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
+    - ${{ if eq(parameters.artifactNames, '') }}:
       - task: DownloadBuildArtifacts@0
         displayName: Download Build Artifacts
         inputs:
@@ -41,20 +54,9 @@ jobs:
           project: $(AzDOProjectName)
           pipeline: $(AzDOPipelineId)
           buildId: $(AzDOBuildId)
-          artifactName: ${{ artifactName }}
+          downloadType: specific files
+          itemPattern: "**"
           downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
-  - ${{ if eq(parameters.artifactNames, '') }}:
-    - task: DownloadBuildArtifacts@0
-      displayName: Download Build Artifacts
-      inputs:
-        buildType: specific
-        buildVersionToDownload: specific
-        project: $(AzDOProjectName)
-        pipeline: $(AzDOPipelineId)
-        buildId: $(AzDOBuildId)
-        downloadType: specific files
-        itemPattern: "**"
-        downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
   - powershell: eng/common/sdl/extract-artifact-packages.ps1
       -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
       -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
@@ -81,7 +83,7 @@ jobs:
       continueOnError: ${{ parameters.sdlContinueOnError }}
   - ${{ if eq(parameters.overrideParameters, '') }}:
     - powershell: eng/common/sdl/execute-all-sdl-tools.ps1
-        -GuardianPackageName Microsoft.Guardian.Cli.0.7.2
+        -GuardianPackageName Microsoft.Guardian.Cli.win10-x64.0.20.1
         -NugetPackageDirectory $(Build.SourcesDirectory)\.packages
         -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
         ${{ parameters.additionalParameters }}

eng/common/templates/job/job.yml

@@ -24,6 +24,7 @@ parameters:
   enablePublishBuildAssets: false
   enablePublishTestResults: false
   enablePublishUsingPipelines: false
+  useBuildManifest: false
   mergeTestResults: false
   testRunTitle: $(AgentOsName)-$(BuildConfiguration)-xunit
   name: ''
@@ -218,3 +219,12 @@ jobs:
         ArtifactName: AssetManifests
       continueOnError: ${{ parameters.continueOnError }}
       condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
+
+  - ${{ if eq(parameters.useBuildManifest, true) }}:
+    - task: PublishBuildArtifacts@1
+      displayName: Publish Build Manifest
+      inputs:
+        PathToPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/manifest.props'
+        PublishLocation: Container
+        ArtifactName: BuildManifests
+      continueOnError: ${{ parameters.continueOnError }}

eng/common/templates/post-build/channels/generic-public-channel.yml

@@ -25,7 +25,7 @@ stages:
   - job: publish_symbols
     displayName: Symbol Publishing
     dependsOn: setupMaestroVars
-    condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }}))
+    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} ))
     variables:
       - group: DotNet-Symbol-Server-Pats
       - name: AzDOProjectName
@@ -99,7 +99,7 @@ stages:
         value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
       - name: ArtifactsCategory
         value: ${{ coalesce(variables._DotNetArtifactsCategory, '.NETCore') }}
-    condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }}))
+    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} ))
     pool:
       vmImage: 'windows-2019'
     steps:

eng/common/templates/post-build/common-variables.yml

@@ -63,7 +63,7 @@ variables:
   - name: MaestroApiAccessToken
     value: $(MaestroAccessToken)
   - name: MaestroApiVersion
-    value: "2019-01-16"
+    value: "2020-02-20"
 
   - name: SourceLinkCLIVersion
     value: 3.0.0

eng/common/templates/post-build/post-build.yml

@@ -9,12 +9,14 @@ parameters:
     continueOnError: false
     params: ''
     artifactNames: ''
+    downloadArtifacts: true
 
   # These parameters let the user customize the call to sdk-task.ps1 for publishing
   # symbols & general artifacts as well as for signing validation
   symbolPublishingAdditionalParameters: ''
   artifactsPublishingAdditionalParameters: ''
   signingValidationAdditionalParameters: ''
+  useBuildManifest: false
 
   # Which stages should finish execution before post-build stages start
   validateDependsOn:
@@ -53,18 +55,16 @@ stages:
     displayName: Post-build Checks
     dependsOn: setupMaestroVars
     variables:
-      - name: InitialChannels
-        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'] ]
-      - name: PromoteToMaestroChannelId
-        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'] ]
+      - name: TargetChannels
+        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
     pool:
       vmImage: 'windows-2019'
     steps:
       - task: PowerShell@2
         displayName: Maestro Channels Consistency
         inputs:
           filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1
-          arguments: -PromoteToChannels "$(InitialChannels)[$(PromoteToMaestroChannelId)]"
+          arguments: -PromoteToChannels "$(TargetChannels)"
             -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview2ChannelId}},${{parameters.Net5Preview3ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}}
 
   - job:
@@ -113,6 +113,16 @@ stages:
     pool:
       vmImage: 'windows-2019'
     steps:
+      - ${{ if eq(parameters.useBuildManifest, true) }}:
+        - task: DownloadBuildArtifacts@0
+          displayName: Download build manifest
+          inputs:
+            buildType: specific
+            buildVersionToDownload: specific
+            project: $(AzDOProjectName)
+            pipeline: $(AzDOPipelineId)
+            buildId: $(AzDOBuildId)
+            artifactName: BuildManifests
       - task: DownloadBuildArtifacts@0
         displayName: Download Package Artifacts
         inputs:
@@ -135,11 +145,13 @@ stages:
           filePath: eng\common\enable-cross-org-publishing.ps1
           arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
 
+      # Signing validation will optionally work with the buildmanifest file which is downloaded from
+      # Azure DevOps above.
       - task: PowerShell@2
         displayName: Validate
         inputs:
           filePath: eng\common\sdk-task.ps1
-          arguments: -task SigningValidation -restore -msbuildEngine dotnet
+          arguments: -task SigningValidation -restore -msbuildEngine vs
             /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts'
             /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt'
             ${{ parameters.signingValidationAdditionalParameters }}
@@ -192,6 +204,7 @@ stages:
       additionalParameters: ${{ parameters.SDLValidationParameters.params }}
       continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }}
       artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }}
+      downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }}
 
 - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
   parameters: