From a2e0141f6b56d0e68b67a9c2a84be117826e0dfb Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Sat, 2 Mar 2019 13:25:57 +0000 Subject: [PATCH 1/3] Update dependencies from https://github.com/dotnet/arcade build 20190301.5 This change updates the following dependencies - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19151.5 - Microsoft.DotNet.Helix.Sdk - 2.0.0-beta.19151.5 --- eng/Version.Details.xml | 8 ++++---- eng/common/tools.ps1 | 2 -- global.json | 4 ++-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 8868a3b18f75..a6fcc5a7795f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -2,13 +2,13 @@ - + https://github.com/dotnet/arcade - 6c034531b2af9e6b2f76c86d471fd308a400269e + 2f6e31ec56a54a5bd1cdd2d9410dae2ddc375d23 - + https://github.com/dotnet/arcade - 6c034531b2af9e6b2f76c86d471fd308a400269e + 2f6e31ec56a54a5bd1cdd2d9410dae2ddc375d23 https://github.com/dotnet/corefx diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index ae33b37d3358..de7523cae581 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -321,12 +321,10 @@ function LocateVisualStudio([object]$vsRequirements = $null){ function InitializeBuildTool() { if (Test-Path variable:global:_BuildTool) { - Write-Host "variable:global:_BuildTool initialized." -ForegroundColor Red return $global:_BuildTool } if (-not $msbuildEngine) { - Write-Host "-not $msbuildEngine" -ForegroundColor Red $msbuildEngine = GetDefaultMSBuildEngine } diff --git a/global.json b/global.json index dac26384fa69..4f975d93bc6d 100644 --- a/global.json +++ b/global.json @@ -7,7 +7,7 @@ "python": "2.7.15" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19128.3", - "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19128.3" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19151.5", + "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19151.5" } } From c57968a493aa23e61b9562966f07da28ae48f73c Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Tue, 5 Mar 2019 13:26:56 +0000 Subject: [PATCH 2/3] Update dependencies from https://github.com/dotnet/arcade build 20190304.14 This change updates the following dependencies - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19154.14 - Microsoft.DotNet.Helix.Sdk - 2.0.0-beta.19154.14 --- eng/Version.Details.xml | 8 +-- eng/common/darc-init.ps1 | 2 +- eng/common/darc-init.sh | 2 +- eng/common/templates/steps/send-to-helix.yml | 53 ++++++++++---------- global.json | 4 +- 5 files changed, 35 insertions(+), 34 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index a6fcc5a7795f..1662aef0d9da 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -2,13 +2,13 @@ - + https://github.com/dotnet/arcade - 2f6e31ec56a54a5bd1cdd2d9410dae2ddc375d23 + 827c7896f8aa902136ad9dc68cb46147a43cd383 - + https://github.com/dotnet/arcade - 2f6e31ec56a54a5bd1cdd2d9410dae2ddc375d23 + 827c7896f8aa902136ad9dc68cb46147a43cd383 https://github.com/dotnet/corefx diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index 24676b261d58..2467ebdd4225 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -19,7 +19,7 @@ function InstallDarcCli ($darcVersion) { # Until we can anonymously query the BAR API for the latest arcade-services # build applied to the PROD channel, this is hardcoded. if (-not $darcVersion) { - $darcVersion = '1.1.0-beta.19120.2' + $darcVersion = '1.1.0-beta.19151.3' } $arcadeServicesSource = 'https://dotnetfeed.blob.core.windows.net/dotnet-arcade/index.json' diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index d4dfdc94fcb0..8d63dd711b2a 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash source="${BASH_SOURCE[0]}" -darcVersion="1.1.0-beta.19120.2" +darcVersion="1.1.0-beta.19151.3" while [[ $# > 0 ]]; do opt="$(echo "$1" | awk '{print tolower($0)}')" diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index 1fbf8b889732..38e143818721 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -1,32 +1,33 @@ parameters: - HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' - HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number - HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/api/2018-03-14/info/queues for a list of queues - HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group - HelixPreCommands: '' # optional -- commands to run before Helix work item execution - HelixPostCommands: '' # optional -- commands to run after Helix work item execution - WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects - WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects - WorkItemTimeout: '' # optional -- a timeout in seconds for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects - CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload - XUnitProjects: '' # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true - XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects - XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner - XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects - IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json - DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json - EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control - WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." - IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set - Creator: '' # optional -- if the build is external, use this to specify who is sending the job - condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() - continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' + HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number + HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/api/2018-03-14/info/queues for a list of queues + HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixPreCommands: '' # optional -- commands to run before Helix work item execution + HelixPostCommands: '' # optional -- commands to run after Helix work item execution + WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects + WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects + WorkItemTimeout: '' # optional -- a timeout in seconds for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects + CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload + XUnitProjects: '' # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true + XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects + XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner + XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects + IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion + DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json + EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control + WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." + IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set + Creator: '' # optional -- if the build is external, use this to specify who is sending the job + DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO + condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() + continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false steps: - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' - displayName: Send job to Helix (Windows) + displayName: ${{ parameters.DisplayNamePrefix }} (Windows) env: BuildConfig: $(_BuildConfig) HelixSource: ${{ parameters.HelixSource }} @@ -53,7 +54,7 @@ steps: condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) continueOnError: ${{ parameters.continueOnError }} - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog - displayName: Send job to Helix (Unix) + displayName: ${{ parameters.DisplayNamePrefix }} (Unix) env: BuildConfig: $(_BuildConfig) HelixSource: ${{ parameters.HelixSource }} diff --git a/global.json b/global.json index 4f975d93bc6d..f7f4650902df 100644 --- a/global.json +++ b/global.json @@ -7,7 +7,7 @@ "python": "2.7.15" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19151.5", - "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19151.5" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19154.14", + "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19154.14" } } From 061a14731b2d7d2a137397385ffa6831fc77903b Mon Sep 17 00:00:00 2001 From: dotnet-maestro <@dotnet-maestro> Date: Fri, 8 Mar 2019 01:16:30 +0000 Subject: [PATCH 3/3] Update dependencies from https://github.com/dotnet/arcade build 20190307.13 This change updates the following dependencies - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19157.13 - Microsoft.DotNet.Helix.Sdk - 2.0.0-beta.19157.13 --- eng/Version.Details.xml | 8 +-- eng/common/build.sh | 9 ++- eng/common/generate-graph-files.ps1 | 61 ++++++++++++++++++++ eng/common/templates/job/job.yml | 4 +- eng/common/templates/steps/send-to-helix.yml | 4 +- global.json | 4 +- 6 files changed, 78 insertions(+), 12 deletions(-) create mode 100644 eng/common/generate-graph-files.ps1 diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 1662aef0d9da..3dfcfa4547c5 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -2,13 +2,13 @@ - + https://github.com/dotnet/arcade - 827c7896f8aa902136ad9dc68cb46147a43cd383 + 4b2966daf2675f9588d27cdb06803dda6d5a7ec5 - + https://github.com/dotnet/arcade - 827c7896f8aa902136ad9dc68cb46147a43cd383 + 4b2966daf2675f9588d27cdb06803dda6d5a7ec5 https://github.com/dotnet/corefx diff --git a/eng/common/build.sh b/eng/common/build.sh index 0227c6e1e554..40b1e8ec73e3 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -137,13 +137,16 @@ while [[ $# > 0 ]]; do node_reuse=$2 shift ;; - /p:*) + -p:*|/p:*) properties="$properties $1" ;; - /m:*) + -m:*|/m:*) properties="$properties $1" ;; - /bl:*) + -bl:*|/bl:*) + properties="$properties $1" + ;; + -dl:*|/dl:*) properties="$properties $1" ;; *) diff --git a/eng/common/generate-graph-files.ps1 b/eng/common/generate-graph-files.ps1 new file mode 100644 index 000000000000..c04c80e4f61d --- /dev/null +++ b/eng/common/generate-graph-files.ps1 @@ -0,0 +1,61 @@ +Param( + [Parameter(Mandatory=$true)][string] $barToken, # Token generated at https://maestro-prod.westus2.cloudapp.azure.com/Account/Tokens + [Parameter(Mandatory=$true)][string] $gitHubPat, # GitHub personal access token from https://github.com/settings/tokens (no auth scopes needed) + [Parameter(Mandatory=$true)][string] $azdoPat, # Azure Dev Ops tokens from https://dev.azure.com/dnceng/_details/security/tokens (code read scope needed) + [Parameter(Mandatory=$true)][string] $outputFolder, # Where the graphviz.txt file will be created + [string] $darcVersion = '1.1.0-beta.19156.4', # darc's version + [switch] $includeToolset # Whether the graph should include toolset dependencies or not. i.e. arcade, optimization. For more about + # toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies +) + +$ErrorActionPreference = "Stop" +. $PSScriptRoot\tools.ps1 + +function CheckExitCode ([string]$stage) +{ + $exitCode = $LASTEXITCODE + if ($exitCode -ne 0) { + Write-Host "Something failed in stage: '$stage'. Check for errors above. Exiting now..." + ExitWithExitCode $exitCode + } +} + +try { + Push-Location $PSScriptRoot + + Write-Host "Installing darc..." + . .\darc-init.ps1 -darcVersion $darcVersion + CheckExitCode "Running darc-init" + + $darcExe = "$env:USERPROFILE\.dotnet\tools" + $darcExe = Resolve-Path "$darcExe\darc.exe" + + Create-Directory $outputFolder + + $graphVizFilePath = "$outputFolder\graphviz.txt" + $graphFilePath = "$outputFolder\graph.txt" + $options = "get-dependency-graph --graphviz '$graphVizFilePath' --github-pat $gitHubPat --azdev-pat $azdoPat --password $barToken --output-file $graphFilePath" + + if ($includeToolset) { + Write-Host "Toolsets will be included in the graph..." + $options += " --include-toolset" + } + + Write-Host "Generating dependency graph..." + $darc = Invoke-Expression "& `"$darcExe`" $options" + CheckExitCode "Generating dependency graph" + + $graph = Get-Content $graphVizFilePath + Set-Content $graphVizFilePath -Value "Paste the following digraph object in http://www.webgraphviz.com `r`n", $graph + Write-Host "'$graphVizFilePath' and '$graphFilePath' created!" +} +catch { + if (!$includeToolset) { + Write-Host "This might be a toolset repo which includes only toolset dependencies. " -NoNewline -ForegroundColor Yellow + Write-Host "Since -includeToolset is not set there is no graph to create. Include -includeToolset and try again..." -ForegroundColor Yellow + } + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} \ No newline at end of file diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index cd4e5731a624..74dd81fdc0a3 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -179,7 +179,7 @@ jobs: continueOnError: true condition: always() - - ${{ if and(eq(parameters.enablePublishBuildAssets, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(variables['_PublishUsingPipelines'], 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: CopyFiles@2 displayName: Gather Asset Manifests inputs: @@ -194,4 +194,4 @@ jobs: PublishLocation: Container ArtifactName: AssetManifests continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) \ No newline at end of file + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index 38e143818721..0925e8ebd123 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -51,6 +51,7 @@ steps: EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) continueOnError: ${{ parameters.continueOnError }} - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog @@ -78,5 +79,6 @@ steps: EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} + continueOnError: ${{ parameters.continueOnError }} \ No newline at end of file diff --git a/global.json b/global.json index f7f4650902df..9e0fcfd2d940 100644 --- a/global.json +++ b/global.json @@ -7,7 +7,7 @@ "python": "2.7.15" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19154.14", - "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19154.14" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19157.13", + "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19157.13" } }