[master] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -2,13 +2,13 @@
 <Dependencies>
   <ProductDependencies></ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19128.3">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19157.13">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>6c034531b2af9e6b2f76c86d471fd308a400269e</Sha>
+      <Sha>4b2966daf2675f9588d27cdb06803dda6d5a7ec5</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.19128.3">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.19157.13">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>6c034531b2af9e6b2f76c86d471fd308a400269e</Sha>
+      <Sha>4b2966daf2675f9588d27cdb06803dda6d5a7ec5</Sha>
     </Dependency>
     <Dependency Name="Microsoft.Private.CoreFx.NETCoreApp" Version="4.6.0-preview4.19151.4">
       <Uri>https://github.com/dotnet/corefx</Uri>

eng/common/build.sh

@@ -137,13 +137,16 @@ while [[ $# > 0 ]]; do
       node_reuse=$2
       shift
       ;;
-    /p:*)
+    -p:*|/p:*)
       properties="$properties $1"
       ;;
-    /m:*)
+    -m:*|/m:*)
       properties="$properties $1"
       ;;
-    /bl:*)
+    -bl:*|/bl:*)
+      properties="$properties $1"
+      ;;
+    -dl:*|/dl:*)
       properties="$properties $1"
       ;;
     *)

eng/common/darc-init.ps1

@@ -19,7 +19,7 @@ function InstallDarcCli ($darcVersion) {
   # Until we can anonymously query the BAR API for the latest arcade-services
   # build applied to the PROD channel, this is hardcoded.
   if (-not $darcVersion) {
-    $darcVersion = '1.1.0-beta.19120.2'
+    $darcVersion = '1.1.0-beta.19151.3'
   }
 
   $arcadeServicesSource = 'https://dotnetfeed.blob.core.windows.net/dotnet-arcade/index.json'

eng/common/darc-init.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 source="${BASH_SOURCE[0]}"
-darcVersion="1.1.0-beta.19120.2"
+darcVersion="1.1.0-beta.19151.3"
 
 while [[ $# > 0 ]]; do
   opt="$(echo "$1" | awk '{print tolower($0)}')"

eng/common/generate-graph-files.ps1

@@ -0,0 +1,61 @@
+Param(
+  [Parameter(Mandatory=$true)][string] $barToken,       # Token generated at https://maestro-prod.westus2.cloudapp.azure.com/Account/Tokens
+  [Parameter(Mandatory=$true)][string] $gitHubPat,      # GitHub personal access token from https://github.com/settings/tokens (no auth scopes needed)
+  [Parameter(Mandatory=$true)][string] $azdoPat,        # Azure Dev Ops tokens from https://dev.azure.com/dnceng/_details/security/tokens (code read scope needed)
+  [Parameter(Mandatory=$true)][string] $outputFolder,   # Where the graphviz.txt file will be created
+  [string] $darcVersion = '1.1.0-beta.19156.4',         # darc's version
+  [switch] $includeToolset                              # Whether the graph should include toolset dependencies or not. i.e. arcade, optimization. For more about
+                                                        # toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies
+)
+
+$ErrorActionPreference = "Stop"
+. $PSScriptRoot\tools.ps1
+
+function CheckExitCode ([string]$stage)
+{
+  $exitCode = $LASTEXITCODE
+  if ($exitCode  -ne 0) {
+    Write-Host "Something failed in stage: '$stage'. Check for errors above. Exiting now..."
+    ExitWithExitCode $exitCode
+  }
+}
+
+try {
+  Push-Location $PSScriptRoot
+
+  Write-Host "Installing darc..."
+  . .\darc-init.ps1 -darcVersion $darcVersion
+  CheckExitCode "Running darc-init"
+
+  $darcExe = "$env:USERPROFILE\.dotnet\tools"
+  $darcExe = Resolve-Path "$darcExe\darc.exe"
+
+  Create-Directory $outputFolder
+
+  $graphVizFilePath = "$outputFolder\graphviz.txt"
+  $graphFilePath = "$outputFolder\graph.txt"
+  $options = "get-dependency-graph --graphviz '$graphVizFilePath' --github-pat $gitHubPat --azdev-pat $azdoPat --password $barToken --output-file $graphFilePath"
+
+  if ($includeToolset) {
+    Write-Host "Toolsets will be included in the graph..."
+    $options += " --include-toolset"
+  }
+
+  Write-Host "Generating dependency graph..."
+  $darc = Invoke-Expression "& `"$darcExe`" $options"
+  CheckExitCode "Generating dependency graph"
+
+  $graph = Get-Content $graphVizFilePath
+  Set-Content $graphVizFilePath -Value "Paste the following digraph object in http://www.webgraphviz.com `r`n", $graph
+  Write-Host "'$graphVizFilePath' and '$graphFilePath' created!"
+}
+catch {
+  if (!$includeToolset) {
+    Write-Host "This might be a toolset repo which includes only toolset dependencies. " -NoNewline -ForegroundColor Yellow
+    Write-Host "Since -includeToolset is not set there is no graph to create. Include -includeToolset and try again..." -ForegroundColor Yellow
+  }
+  Write-Host $_
+  Write-Host $_.Exception
+  Write-Host $_.ScriptStackTrace
+  ExitWithExitCode 1
+}

eng/common/templates/job/job.yml

@@ -179,7 +179,7 @@ jobs:
       continueOnError: true
       condition: always()
 
-  - ${{ if and(eq(parameters.enablePublishBuildAssets, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+  - ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(variables['_PublishUsingPipelines'], 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
     - task: CopyFiles@2
       displayName: Gather Asset Manifests
       inputs:
@@ -194,4 +194,4 @@ jobs:
         PublishLocation: Container
         ArtifactName: AssetManifests
       continueOnError: ${{ parameters.continueOnError }}
-      condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
+      condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))

eng/common/templates/steps/send-to-helix.yml

@@ -1,32 +1,33 @@
 parameters:
-  HelixSource: 'pr/default'          # required -- sources must start with pr/, official/, prodcon/, or agent/
-  HelixType: 'tests/default/'        # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/'
-  HelixBuild: $(Build.BuildNumber)   # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number
-  HelixTargetQueues: ''              # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/api/2018-03-14/info/queues for a list of queues
-  HelixAccessToken: ''               # required -- access token to make Helix API requests; should be provided by the appropriate variable group
-  HelixPreCommands: ''               # optional -- commands to run before Helix work item execution
-  HelixPostCommands: ''              # optional -- commands to run after Helix work item execution
-  WorkItemDirectory: ''              # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects
-  WorkItemCommand: ''                # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects
-  WorkItemTimeout: ''                # optional -- a timeout in seconds for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects
-  CorrelationPayloadDirectory: ''    # optional -- a directory to zip up and send to Helix as a correlation payload
-  XUnitProjects: ''                  # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true
-  XUnitPublishTargetFramework: ''    # optional -- framework to use to publish your xUnit projects
-  XUnitRuntimeTargetFramework: ''    # optional -- framework to use for the xUnit console runner
-  XUnitRunnerVersion: ''             # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects
-  IncludeDotNetCli: false            # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
-  DotNetCliPackageType: ''           # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
-  DotNetCliVersion: ''               # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
-  EnableXUnitReporter: false         # optional -- true enables XUnit result reporting to Mission Control
-  WaitForWorkItemCompletion: true    # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
-  IsExternal: false                  # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set
-  Creator: ''                        # optional -- if the build is external, use this to specify who is sending the job
-  condition: succeeded()             # optional -- condition for step to execute; defaults to succeeded()
-  continueOnError: false             # optional -- determines whether to continue the build if the step errors; defaults to false
+  HelixSource: 'pr/default'              # required -- sources must start with pr/, official/, prodcon/, or agent/
+  HelixType: 'tests/default/'            # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/'
+  HelixBuild: $(Build.BuildNumber)       # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number
+  HelixTargetQueues: ''                  # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/api/2018-03-14/info/queues for a list of queues
+  HelixAccessToken: ''                   # required -- access token to make Helix API requests; should be provided by the appropriate variable group
+  HelixPreCommands: ''                   # optional -- commands to run before Helix work item execution
+  HelixPostCommands: ''                  # optional -- commands to run after Helix work item execution
+  WorkItemDirectory: ''                  # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects
+  WorkItemCommand: ''                    # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects
+  WorkItemTimeout: ''                    # optional -- a timeout in seconds for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects
+  CorrelationPayloadDirectory: ''        # optional -- a directory to zip up and send to Helix as a correlation payload
+  XUnitProjects: ''                      # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true
+  XUnitPublishTargetFramework: ''        # optional -- framework to use to publish your xUnit projects
+  XUnitRuntimeTargetFramework: ''        # optional -- framework to use for the xUnit console runner
+  XUnitRunnerVersion: ''                 # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects
+  IncludeDotNetCli: false                # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
+  DotNetCliPackageType: ''               # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
+  DotNetCliVersion: ''                   # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
+  EnableXUnitReporter: false             # optional -- true enables XUnit result reporting to Mission Control
+  WaitForWorkItemCompletion: true        # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
+  IsExternal: false                      # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set
+  Creator: ''                            # optional -- if the build is external, use this to specify who is sending the job
+  DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO 
+  condition: succeeded()                 # optional -- condition for step to execute; defaults to succeeded()
+  continueOnError: false                 # optional -- determines whether to continue the build if the step errors; defaults to false
 
 steps:
   - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"'
-    displayName: Send job to Helix (Windows)
+    displayName: ${{ parameters.DisplayNamePrefix }} (Windows)
     env:
       BuildConfig: $(_BuildConfig)
       HelixSource: ${{ parameters.HelixSource }}
@@ -50,10 +51,11 @@ steps:
       EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
       WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
       Creator: ${{ parameters.Creator }}
+      SYSTEM_ACCESSTOKEN: $(System.AccessToken)
     condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT'))
     continueOnError: ${{ parameters.continueOnError }}
   - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog
-    displayName: Send job to Helix (Unix)
+    displayName: ${{ parameters.DisplayNamePrefix }} (Unix)
     env:
       BuildConfig: $(_BuildConfig)
       HelixSource: ${{ parameters.HelixSource }}
@@ -77,5 +79,6 @@ steps:
       EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
       WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
       Creator: ${{ parameters.Creator }}
+      SYSTEM_ACCESSTOKEN: $(System.AccessToken)
     condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT'))
-    continueOnError: ${{ parameters.continueOnError }}
+    continueOnError: ${{ parameters.continueOnError }}

eng/common/tools.ps1

@@ -321,12 +321,10 @@ function LocateVisualStudio([object]$vsRequirements = $null){
 
 function InitializeBuildTool() {
   if (Test-Path variable:global:_BuildTool) {
-    Write-Host "variable:global:_BuildTool initialized." -ForegroundColor Red
     return $global:_BuildTool
   }
 
   if (-not $msbuildEngine) {
-    Write-Host "-not $msbuildEngine" -ForegroundColor Red
     $msbuildEngine = GetDefaultMSBuildEngine
   }
 

global.json

@@ -7,7 +7,7 @@
     "python": "2.7.15"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19128.3",
-    "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19128.3"
+    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19157.13",
+    "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19157.13"
   }
 }