From 2a27fae54d79c6201af3d6427ef92ffe10b378a2 Mon Sep 17 00:00:00 2001 From: Tomas Matousek Date: Sun, 2 Dec 2018 10:56:02 -0800 Subject: [PATCH 1/9] Bootstrapper improvements. Replace & PS operator with custom process launcher. The & operator does not allow to send colorized output to the host without polluting function return value. Automatically terminate script when MSBuild fails. Memoize results of Initialize* functions. Move build customization hooks to build.ps1. Make binary log off by default since it slows down build. It's on in CI build. Fixes to quiet restore workaround. Add set -u to bash script. --- .editorconfig | 6 + eng/common/build.ps1 | 47 +++- eng/common/build.sh | 184 ++++++++------- eng/common/tools.ps1 | 221 ++++++++++++------ eng/common/tools.sh | 84 +++---- .../tools/Build.proj | 31 ++- .../tools/RepoLayout.props | 2 +- 7 files changed, 364 insertions(+), 211 deletions(-) diff --git a/.editorconfig b/.editorconfig index f45a09ce054..746feabd043 100644 --- a/.editorconfig +++ b/.editorconfig @@ -7,6 +7,12 @@ indent_style = space [*.{sh}] end_of_line = lf +indent_size = 2 +tab_width = 2 + +[*.{ps1}] +indent_size = 2 +tab_width = 2 [*.{cs}] indent_size = 4 diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 76f108fd5d8..9a1a75e0272 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -19,6 +19,7 @@ Param( [switch] $pack, [switch] $publish, [switch] $publishBuildAssets, + [switch][Alias('bl')]$binaryLog, [switch] $ci, [switch] $prepareMachine, [switch] $help, @@ -31,6 +32,7 @@ function Print-Usage() { Write-Host "Common settings:" Write-Host " -configuration Build configuration Debug, Release" Write-Host " -verbosity Msbuild verbosity (q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic])" + Write-Host " -binaryLog Output binary log" Write-Host " -help Print help and exit" Write-Host "" @@ -64,17 +66,45 @@ if ($help -or (($properties -ne $null) -and ($properties.Contains("/help") -or $ exit 0 } +function ConfigureToolset { + # Include custom tools configuration + $script = Join-Path $EngRoot "configure-toolset.ps1" + + if (Test-Path $script) { + . $script + } +} + +function InitializeCustomToolset { + if (-not $restore) { + return + } + + $script = Join-Path $EngRoot "restore-toolset.ps1" + + if (Test-Path $script) { + . $script + } +} + try { if ($projects -eq "") { $projects = Join-Path $RepoRoot "*.sln" } - InitializeTools + if ($ci) { + $binaryLog = $true + } + + ConfigureToolset + $toolsetBuildProj = InitializeToolset + InitializeCustomToolset - $BuildLog = Join-Path $LogDir "Build.binlog" + $buildLog = Join-Path $LogDir "Build.binlog" + $bl = if ($binaryLog) { "/bl:$buildLog" } else { "" } - MSBuild $ToolsetBuildProj ` - /bl:$BuildLog ` + MSBuild $toolsetBuildProj ` + $bl ` /p:Configuration=$configuration ` /p:Projects=$projects ` /p:RepoRoot=$RepoRoot ` @@ -92,13 +122,6 @@ try { /p:Execute=$execute ` /p:ContinuousIntegrationBuild=$ci ` @properties - - if ($lastExitCode -ne 0) { - Write-Host "Build Failed (exit code '$lastExitCode'). See log: $BuildLog" -ForegroundColor Red - ExitWithExitCode $lastExitCode - } - - ExitWithExitCode $lastExitCode } catch { Write-Host $_ @@ -106,3 +129,5 @@ catch { Write-Host $_.ScriptStackTrace ExitWithExitCode 1 } + +ExitWithExitCode 0 diff --git a/eng/common/build.sh b/eng/common/build.sh index 941db3bd570..1f8dc49df38 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -1,5 +1,35 @@ #!/usr/bin/env bash +# Stop script if unbound variable found (use ${var:-} if intentional) +set -u + +usage() +{ + echo "Common settings:" + echo " --configuration Build configuration: 'Debug' or 'Release' (short: --c)" + echo " --verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" + echo " --binaryLog Create MSBuild binary log (short: -bl)" + echo "" + echo "Actions:" + echo " --restore Restore dependencies (short: -r)" + echo " --build Build all projects (short: -b)" + echo " --rebuild Rebuild all projects" + echo " --test Run all unit tests (short: -t)" + echo " --sign Sign build outputs" + echo " --publish Publish artifacts (e.g. symbols)" + echo " --pack Package build outputs into NuGet packages and Willow components" + echo " --help Print help and exit (short: -h)" + echo "" + echo "Advanced settings:" + echo " --projects Project or solution file(s) to build" + echo " --ci Set when running on CI server" + echo " --prepareMachine Prepare machine for CI run, clean up processes after build" + echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" + echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" + echo "" + echo "Command line arguments starting with '/p:' are passed through to MSBuild." +} + source="${BASH_SOURCE[0]}" # resolve $source until the file is no longer a symlink @@ -27,6 +57,7 @@ ci=false warnaserror=true nodereuse=true +binary_log=false projects='' configuration='Debug' @@ -34,119 +65,122 @@ prepare_machine=false verbosity='minimal' properties='' -while (($# > 0)); do - lowerI="$(echo $1 | awk '{print tolower($0)}')" - case $lowerI in - --build) - build=true - shift 1 - ;; - --ci) - ci=true - shift 1 - ;; - --configuration) - configuration=$2 - shift 2 - ;; - --help) - echo "Common settings:" - echo " --configuration Build configuration Debug, Release" - echo " --verbosity Msbuild verbosity (q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic])" - echo " --help Print help and exit" - echo "" - echo "Actions:" - echo " --restore Restore dependencies" - echo " --build Build solution" - echo " --rebuild Rebuild solution" - echo " --test Run all unit tests in the solution" - echo " --sign Sign build outputs" - echo " --publish Publish artifacts (e.g. symbols)" - echo " --pack Package build outputs into NuGet packages and Willow components" - echo "" - echo "Advanced settings:" - echo " --solution Path to solution to build" - echo " --ci Set when running on CI server" - echo " --prepareMachine Prepare machine for CI run" - echo "" - echo "Command line arguments not listed above are passed through to MSBuild." +while [[ $# > 0 ]]; do + opt="$(echo "$1" | awk '{print tolower($0)}')" + case "$opt" in + --help|-h) + usage exit 0 ;; - --pack) - pack=true - shift 1 + --configuration|-c) + configuration=$2 + shift ;; - --preparemachine) - prepare_machine=true - shift 1 + --verbosity|-v) + verbosity=$2 + shift ;; - --rebuild) - rebuild=true - shift 1 + --binarylog|-bl) + binary_log=true ;; - --restore) + --restore|-r) restore=true - shift 1 ;; - --sign) - sign=true - shift 1 + --build|-b) + build=true ;; - --solution) - solution=$2 - shift 2 + --rebuild) + rebuild=true ;; - --projects) - projects=$2 - shift 2 + --pack) + pack=true ;; - --test) + --test|-t) test=true - shift 1 ;; --integrationtest) integration_test=true - shift 1 ;; --performancetest) performance_test=true - shift 1 + ;; + --sign) + sign=true ;; --publish) publish=true - shift 1 ;; - --verbosity) - verbosity=$2 - shift 2 + --preparemachine) + prepare_machine=true + ;; + --projects) + projects=$2 + shift + ;; + --ci) + ci=true ;; --warnaserror) warnaserror=$2 - shift 2 + shift ;; --nodereuse) nodereuse=$2 - shift 2 + shift ;; - *) + /p:*) properties="$properties $1" - shift 1 + ;; + *) + echo "Invalid argument: $1" + usage + exit 1 ;; esac + + shift done . "$scriptroot/tools.sh" +function ConfigureToolset { + local script="$eng_root/configure-toolset.sh" + + if [[ -a "$script" ]]; then + . "$script" + fi +} + +function InitializeCustomToolset { + local script="$eng_root/restore-toolset.sh" + + if [[ -a "$script" ]]; then + . "$script" + fi +} + if [[ -z $projects ]]; then projects="$repo_root/*.sln" fi -InitializeTools +if [[ "$ci" == true ]]; then + $binary_log = true +fi + +ConfigureToolset +InitializeToolset +InitializeCustomToolset build_log="$log_dir/Build.binlog" -MSBuild "$toolset_build_proj" \ - /bl:"$build_log" \ +if [[ "$binary_log" == true ]]; then + bl="/bl:\"$build_log\"" +else + bl="" +fi + +MSBuild $_InitializeToolset \ + $bl \ /p:Configuration=$configuration \ /p:Projects="$projects" \ /p:RepoRoot="$repo_root" \ @@ -162,10 +196,4 @@ MSBuild "$toolset_build_proj" \ /p:ContinuousIntegrationBuild=$ci \ $properties -lastexitcode=$? - -if [[ $lastexitcode != 0 ]]; then - echo "Build failed (exit code '$lastexitcode'). See log: $build_log" -fi - -ExitWithExitCode $lastexitcode +ExitWithExitCode 0 diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 290637c5fab..aae9c65c1e8 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -2,6 +2,7 @@ $ci = if (Test-Path variable:ci) { $ci } else { $false } $configuration = if (Test-Path variable:configuration) { $configuration } else { "Debug" } +$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci } $nodereuse = if (Test-Path variable:nodereuse) { $nodereuse } else { !$ci } $prepareMachine = if (Test-Path variable:prepareMachine) { $prepareMachine } else { $false } $restore = if (Test-Path variable:restore) { $restore } else { $true } @@ -10,6 +11,8 @@ $warnaserror = if (Test-Path variable:warnaserror) { $warnaserror } else { $true $msbuildEngine = if (Test-Path variable:msbuildEngine) { $msbuildEngine } else { $null } $useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true } +$ProcessesToStopOnExit = @("msbuild", "dotnet", "vbcscompiler") + set-strictmode -version 2.0 $ErrorActionPreference = "Stop" [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 @@ -25,7 +28,43 @@ function Unzip([string]$zipfile, [string]$outpath) { [System.IO.Compression.ZipFile]::ExtractToDirectory($zipfile, $outpath) } +# This will exec a process using the console and return it's exit code. +# This will not throw when the process fails. +# Returns process exit code. +function Exec-Process([string]$command, [string]$commandArgs) { + $startInfo = New-Object System.Diagnostics.ProcessStartInfo + $startInfo.FileName = $command + $startInfo.Arguments = $commandArgs + $startInfo.UseShellExecute = $false + $startInfo.WorkingDirectory = Get-Location + + $process = New-Object System.Diagnostics.Process + $process.StartInfo = $startInfo + $process.Start() | Out-Null + + $finished = $false + try { + while (-not $process.WaitForExit(100)) { + # Non-blocking loop done to allow ctr-c interrupts + } + + $finished = $true + return $global:LASTEXITCODE = $process.ExitCode + } + finally { + # If we didn't finish then an error occured or the user hit ctrl-c. Either + # way kill the process + if (-not $finished) { + $process.Kill() + } + } +} + function InitializeDotNetCli([bool]$install) { + if (Test-Path global:_DotNetInstallDir) { + return $global:_DotNetInstallDir + } + # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism $env:DOTNET_MULTILEVEL_LOOKUP=0 @@ -39,7 +78,10 @@ function InitializeDotNetCli([bool]$install) { # Find the first path on %PATH% that contains the dotnet.exe if ($useInstalledDotNetCli -and ($env:DOTNET_INSTALL_DIR -eq $null)) { - $env:DOTNET_INSTALL_DIR = ${env:PATH}.Split(';') | where { ($_ -ne "") -and (Test-Path (Join-Path $_ "dotnet.exe")) } + $dotnetCmd = Get-Command "dotnet.exe" -ErrorAction SilentlyContinue + if ($dotnetCmd -ne $null) { + $env:DOTNET_INSTALL_DIR = Split-Path $dotnetCmd.Path -Parent + } } $dotnetSdkVersion = $GlobalJson.tools.dotnet @@ -50,9 +92,8 @@ function InitializeDotNetCli([bool]$install) { $dotnetRoot = $env:DOTNET_INSTALL_DIR } else { $dotnetRoot = Join-Path $RepoRoot ".dotnet" - $env:DOTNET_INSTALL_DIR = $dotnetRoot - if (-not (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) { + if (-not (Test-Path(Join-Path $dotnetRoot "sdk\$dotnetSdkVersion"))) { if ($install) { InstallDotNetSdk $dotnetRoot $dotnetSdkVersion } else { @@ -60,9 +101,11 @@ function InitializeDotNetCli([bool]$install) { ExitWithExitCode 1 } } + + $env:DOTNET_INSTALL_DIR = $dotnetRoot } - return $dotnetRoot + return $global:_DotNetInstallDir = $dotnetRoot } function GetDotNetInstallScript([string] $dotnetRoot) { @@ -95,7 +138,11 @@ function InstallDotNetSdk([string] $dotnetRoot, [string] $version) { # Returns full path to msbuild.exe. # Throws on failure. # -function InitializeVisualStudioMSBuild { +function InitializeVisualStudioMSBuild([bool]$install) { + if (Test-Path global:_MSBuildExe) { + return $global:_MSBuildExe + } + $vsMinVersionStr = if (!$GlobalJson.tools.vs.version) { $GlobalJson.tools.vs.version } else { "15.9" } $vsMinVersion = [Version]::new($vsMinVersionStr) @@ -104,7 +151,7 @@ function InitializeVisualStudioMSBuild { $msbuildCmd = Get-Command "msbuild.exe" -ErrorAction SilentlyContinue if ($msbuildCmd -ne $null) { if ($msbuildCmd.Version -ge $vsMinVersion) { - return $msbuildCmd.Path + return $global:_MSBuildExe = $msbuildCmd.Path } # Report error - the developer environment is initialized with incompatible VS version. @@ -119,7 +166,7 @@ function InitializeVisualStudioMSBuild { $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0] InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion - } else { + } elseif ($install) { if (Get-Member -InputObject $GlobalJson.tools -Name "xcopy-msbuild") { $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild' @@ -130,10 +177,12 @@ function InitializeVisualStudioMSBuild { } $vsInstallDir = InstallXCopyMSBuild $xcopyMSBuildVersion + } else { + throw "Unable to find Visual Studio that has required version and components installed" } $msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" } - return Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin\msbuild.exe" + return $global:_MSBuildExe = Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin\msbuild.exe" } function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [string] $vsMajorVersion) { @@ -216,72 +265,71 @@ function LocateVisualStudio { return $vsInfo[0] } -function ConfigureTools { - # Include custom tools configuration - $script = Join-Path $EngRoot "configure-toolset.ps1" - - if (Test-Path $script) { - . $script +function InitializeBuildTool() { + if (Test-Path global:_BuildTool) { + return $global:_BuildTool } -} -function InitializeTools() { - ConfigureTools - - $tools = $GlobalJson.tools + if (-not $msbuildEngine) { + $msbuildEngine = GetDefaultMSBuildEngine + } # Initialize dotnet cli if listed in 'tools' $dotnetRoot = $null - if (Get-Member -InputObject $tools -Name "dotnet") { + if (Get-Member -InputObject $GlobalJson.tools -Name "dotnet") { $dotnetRoot = InitializeDotNetCli -install:$restore } - if (-not $msbuildEngine) { - # Presence of tools.vs indicates the repo needs to build using VS msbuild on Windows. - if (Get-Member -InputObject $tools -Name "vs") { - $msbuildEngine = "vs" - } elseif ($dotnetRoot -ne $null) { - $msbuildEngine = "dotnet" - } else { - Write-Host "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." -ForegroundColor Red - ExitWithExitCode 1 - } - } - if ($msbuildEngine -eq "dotnet") { if (!$dotnetRoot) { Write-Host "/global.json must specify 'tools.dotnet'." -ForegroundColor Red ExitWithExitCode 1 } - $script:buildDriver = Join-Path $dotnetRoot "dotnet.exe" - $script:buildArgs = "msbuild" + $buildTool = @{ Path = Join-Path $dotnetRoot "dotnet.exe"; Command = "msbuild" } } elseif ($msbuildEngine -eq "vs") { try { - $script:buildDriver = InitializeVisualStudioMSBuild - $script:buildArgs = "" - } catch { + $msbuildPath = InitializeVisualStudioMSBuild -install:$restore + } catch { Write-Host $_ -ForegroundColor Red ExitWithExitCode 1 } + + $buildTool = @{ Path = $msbuildPath; Command = "" } } else { Write-Host "Unexpected value of -msbuildEngine: '$msbuildEngine'." -ForegroundColor Red ExitWithExitCode 1 } - InitializeToolSet $script:buildDriver $script:buildArgs - InitializeCustomToolset + return $global:_BuildTool = $buildTool } -function InitializeToolset([string] $buildDriver, [string]$buildArgs) { +function GetDefaultMSBuildEngine() { + # Presence of tools.vs indicates the repo needs to build using VS msbuild on Windows. + if (Get-Member -InputObject $GlobalJson.tools -Name "vs") { + return "vs" + } + + if (Get-Member -InputObject $GlobalJson.tools -Name "dotnet") { + return "dotnet" + } + + Write-Host "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." -ForegroundColor Red + ExitWithExitCode 1 +} + +function InitializeToolset() { + if (Test-Path global:_ToolsetBuildProj) { + return $global:_ToolsetBuildProj + } + $toolsetVersion = $GlobalJson.'msbuild-sdks'.'Microsoft.DotNet.Arcade.Sdk' $toolsetLocationFile = Join-Path $ToolsetDir "$toolsetVersion.txt" if (Test-Path $toolsetLocationFile) { $path = Get-Content $toolsetLocationFile -TotalCount 1 if (Test-Path $path) { - $script:ToolsetBuildProj = $path - return + return $global:_ToolsetBuildProj = $path } } @@ -290,35 +338,20 @@ function InitializeToolset([string] $buildDriver, [string]$buildArgs) { ExitWithExitCode 1 } - $ToolsetRestoreLog = Join-Path $LogDir "ToolsetRestore.binlog" - $proj = Join-Path $ToolsetDir "restore.proj" + $buildTool = InitializeBuildTool + $proj = Join-Path $ToolsetDir "restore.proj" + $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "ToolsetRestore.binlog") } else { "" } + '' | Set-Content $proj - MSBuild $proj /t:__WriteToolsetLocation /clp:None /bl:$ToolsetRestoreLog /p:__ToolsetLocationOutputFile=$toolsetLocationFile - - if ($lastExitCode -ne 0) { - Write-Host "Failed to restore toolset (exit code '$lastExitCode'). See log: $ToolsetRestoreLog" -ForegroundColor Red - ExitWithExitCode $lastExitCode - } - + MSBuild $proj $bl /t:__WriteToolsetLocation /noconsolelogger /p:__ToolsetLocationOutputFile=$toolsetLocationFile + $path = Get-Content $toolsetLocationFile -TotalCount 1 if (!(Test-Path $path)) { throw "Invalid toolset path: $path" } - - $script:ToolsetBuildProj = $path -} - -function InitializeCustomToolset { - if (-not $restore) { - return - } - - $script = Join-Path $EngRoot "restore-toolset.ps1" - - if (Test-Path $script) { - . $script - } + + return $global:_ToolsetBuildProj = $path } function ExitWithExitCode([int] $exitCode) { @@ -330,14 +363,60 @@ function ExitWithExitCode([int] $exitCode) { function Stop-Processes() { Write-Host "Killing running build processes..." - Get-Process -Name "msbuild" -ErrorAction SilentlyContinue | Stop-Process - Get-Process -Name "dotnet" -ErrorAction SilentlyContinue | Stop-Process - Get-Process -Name "vbcscompiler" -ErrorAction SilentlyContinue | Stop-Process + foreach ($processName in $ProcessesToStopOnExit) { + Get-Process -Name $processName -ErrorAction SilentlyContinue | Stop-Process + } +} + +# +# Executes msbuild (or 'dotnet msbuild') with arguments passed to the function. +# The arguments are automatically quoted. +# Terminates the script if the build fails. +# +function MSBuild() { + $buildTool = InitializeBuildTool + + $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodereuse" + + if ($warnaserror) { + $cmdArgs += " /warnaserror /p:TreatWarningsAsErrors=true" + } + + foreach ($arg in $args) { + if ($arg -ne $null -and $arg.Trim() -ne "") { + $cmdArgs += " `"$arg`"" + } + } + + $exitCode = Exec-Process $buildTool.Path $cmdArgs + + if ($exitCode -ne 0) { + Write-Host "Build failed." -ForegroundColor Red + + $buildLog = GetMSBuildBinaryLogCommandLineArgument $args + if ($buildLog -ne $null) { + Write-Host "See log: $buildLog" -ForegroundColor DarkGray + } + + ExitWithExitCode $exitCode + } } -function MsBuild() { - $warnaserrorSwitch = if ($warnaserror) { "/warnaserror" } else { "" } - & $buildDriver $buildArgs $warnaserrorSwitch /m /nologo /clp:Summary /v:$verbosity /nr:$nodereuse $args +function GetMSBuildBinaryLogCommandLineArgument($arguments) { + foreach ($argument in $arguments) { + if ($argument -ne $null) { + $arg = $argument.Trim() + if ($arg.StartsWith("/bl:", "OrdinalIgnoreCase")) { + return $arg.Substring("/bl:".Length) + } + + if ($arg.StartsWith("/binaryLogger:", "OrdinalIgnoreCase")) { + return $arg.Substring("/binaryLogger:".Length) + } + } + } + + return $null } $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..") diff --git a/eng/common/tools.sh b/eng/common/tools.sh index c7d03ee028b..42c7c163e5c 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -5,13 +5,19 @@ set -u ci=${ci:-false} configuration=${configuration:-'Debug'} -nodereuse=${nodereuse:-true} +binary_log=${binary_log:-$ci} prepare_machine=${prepare_machine:-false} restore=${restore:-true} verbosity=${verbosity:-'minimal'} warnaserror=${warnaserror:-true} useInstalledDotNetCli=${useInstalledDotNetCli:-true} +if [[ "$ci" == true ]]; then + nodereuse=${nodereuse:-false} +else + nodereuse=${nodereuse:-true} +fi + repo_root="$scriptroot/../.." eng_root="$scriptroot/.." artifacts_dir="$repo_root/artifacts" @@ -20,8 +26,6 @@ log_dir="$artifacts_dir/log/$configuration" temp_dir="$artifacts_dir/tmp/$configuration" global_json_file="$repo_root/global.json" -build_driver="" -toolset_build_proj="" function ResolvePath { local path=$1 @@ -57,6 +61,10 @@ function ReadGlobalVersion { } function InitializeDotNetCli { + if [[ -n "${_InitializeDotNetCli:-}" ]]; then + return + fi + local install=$1 # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism @@ -65,6 +73,10 @@ function InitializeDotNetCli { # Disable first run since we want to control all package sources export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 + # LTTNG is the logging infrastructure used by Core CLR. Need this variable set + # so it doesn't output warnings to the console. + export LTTNG_HOME="$HOME" + # Source Build uses DotNetCoreSdkDir variable if [[ -n "${DotNetCoreSdkDir:-}" ]]; then export DOTNET_INSTALL_DIR="$DotNetCoreSdkDir" @@ -143,7 +155,22 @@ function GetDotNetInstallScript { _GetDotNetInstallScript="$install_script" } +function InitializeBuildTool { + if [[ -n "${_InitializeBuildTool:-}" ]]; then + return + fi + + InitializeDotNetCli $restore + + # return value + _InitializeBuildTool="$_InitializeDotNetCli/dotnet" +} + function InitializeToolset { + if [[ -n "${_InitializeToolset:-}" ]]; then + return + fi + ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk" local toolset_version=$_ReadGlobalVersion @@ -152,7 +179,8 @@ function InitializeToolset { if [[ -a "$toolset_location_file" ]]; then local path=`cat "$toolset_location_file"` if [[ -a "$path" ]]; then - toolset_build_proj="$path" + # return value + _InitializeToolset="$path" return fi fi @@ -166,47 +194,17 @@ function InitializeToolset { local proj="$toolset_dir/restore.proj" echo '' > "$proj" + MSBuild "$proj" /t:__WriteToolsetLocation /noconsolelogger /bl:"$toolset_restore_log" /p:__ToolsetLocationOutputFile="$toolset_location_file" - MSBuild "$proj" /t:__WriteToolsetLocation /clp:None /bl:"$toolset_restore_log" /p:__ToolsetLocationOutputFile="$toolset_location_file" - local lastexitcode=$? - - if [[ $lastexitcode != 0 ]]; then - echo "Failed to restore toolset (exit code '$lastexitcode'). See log: $toolset_restore_log" >&2 - ExitWithExitCode $lastexitcode - fi - - toolset_build_proj=`cat "$toolset_location_file"` + local toolset_build_proj=`cat "$toolset_location_file"` if [[ ! -a "$toolset_build_proj" ]]; then echo "Invalid toolset path: $toolset_build_proj" >&2 ExitWithExitCode 3 fi -} - -function InitializeCustomToolset { - local script="$eng_root/restore-toolset.sh" - - if [[ -a "$script" ]]; then - . "$script" - fi -} - -function ConfigureTools { - local script="$eng_root/configure-toolset.sh" - - if [[ -a "$script" ]]; then - . "$script" - fi -} - -function InitializeTools { - ConfigureTools - InitializeDotNetCli $restore - build_driver="$_InitializeDotNetCli/dotnet" - - InitializeToolset - InitializeCustomToolset + # return value + _InitializeToolset="$toolset_build_proj" } function ExitWithExitCode { @@ -224,14 +222,20 @@ function StopProcesses { } function MSBuild { + InitializeBuildTool + local warnaserror_switch="" if [[ $warnaserror == true ]]; then warnaserror_switch="/warnaserror" fi - "$build_driver" msbuild /m /nologo /clp:Summary /v:$verbosity /nr:$nodereuse $warnaserror_switch "$@" + "$_InitializeBuildTool" msbuild /m /nologo /clp:Summary /v:$verbosity /nr:$nodereuse $warnaserror_switch /p:TreatWarningsAsErrors=$warnaserror "$@" + lastexitcode=$? - return $? + if [[ $lastexitcode != 0 ]]; then + echo "Build failed (exit code '$lastexitcode')." >&2 + ExitWithExitCode $lastexitcode + fi } # HOME may not be defined in some scenarios, but it is required by NuGet diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/Build.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/Build.proj index 558f9fd71d3..6b95b120e61 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/Build.proj +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/Build.proj @@ -23,7 +23,6 @@ ContinuousIntegrationBuild "true" when building on a CI server (PR build or official build) Restore "true" to restore toolset and solution - QuietRestore "true" to suppress Restore output. Build "true" to build solution Rebuild "true" to rebuild solution Execute "true" to execute an sdk project "execute" target @@ -34,6 +33,10 @@ Pack "true" to build NuGet packages and VS insertion manifests Sign "true" to sign built binaries Publish "true" to publish artifacts (e.g. symbols) + + Workaround for https://github.com/NuGet/Home/issues/4695 + QuietRestore "true" to suppress Restore output. + QuietRestoreBinaryLog "true" to produce binary log for Restore. --> <_SolutionBuildTargets Include="Execute" Condition="'$(Execute)' == 'true' and '$(SdkTaskProjects)' != ''" /> + <_CommonProps Include="Configuration=$(Configuration)"/> <_CommonProps Include="ContinuousIntegrationBuild=$(ContinuousIntegrationBuild)"/> @@ -145,13 +149,20 @@ - <_SolutionBuildPropsArgs Include="@(_SolutionBuildProps->'/p:%(Identity)')" /> - <_RestoreToolsPropArgs Include="@(_RestoreToolsProps->'/p:%(Identity)')" /> + <_SolutionBuildPropsArgs Include="@(_SolutionBuildProps->'/p:%(Identity)\')" Condition="$([MSBuild]::ValueOrDefault(%(_SolutionBuildProps.Identity), '').EndsWith('\'))" /> + <_SolutionBuildPropsArgs Include="@(_SolutionBuildProps->'/p:%(Identity)')" Condition="!$([MSBuild]::ValueOrDefault(%(_SolutionBuildProps.Identity), '').EndsWith('\'))" /> + <_RestoreToolsPropArgs Include="@(_RestoreToolsProps->'/p:%(Identity)\')" Condition="$([MSBuild]::ValueOrDefault(%(_RestoreToolsProps.Identity), '').EndsWith('\'))" /> + <_RestoreToolsPropArgs Include="@(_RestoreToolsProps->'/p:%(Identity)')" Condition="!$([MSBuild]::ValueOrDefault(%(_RestoreToolsProps.Identity), '').EndsWith('\'))" /> - <_SolutionBuildPropsCmdLine>@(_SolutionBuildPropsArgs, ' ') - <_RestoreToolsPropsCmdLine>@(_RestoreToolsPropArgs, ' ') + <_SolutionBuildPropsCmdLine>"@(_SolutionBuildPropsArgs, '" "')" + <_RestoreToolsPropsCmdLine>"@(_RestoreToolsPropArgs, '" "')" + + + + <_SolutionBuildPropsCmdLine>$(_SolutionBuildPropsCmdLine) /bl:"$(ArtifactsLogDir)RestoreRepoTools.binlog" + <_RestoreToolsPropsCmdLine>$(_RestoreToolsPropsCmdLine) /bl:"$(ArtifactsLogDir)Restore.binlog" $(NUGET_PACKAGES) $(UserProfile)\.nuget\packages\ - $([System.Environment]::GetFolderPath(SpecialFolder.Personal))\.nuget\packages\ + $(HOME)/.nuget/packages/ $(NuGetPackageRoot)\ From 615449445ed8f25efcd8ceabcf4b83e6422f322f Mon Sep 17 00:00:00 2001 From: Tomas Matousek Date: Sun, 2 Dec 2018 11:29:31 -0800 Subject: [PATCH 2/9] Disable .NET Core telemetry on CI. --- eng/common/build.sh | 1 - eng/common/tools.ps1 | 5 +++++ eng/common/tools.sh | 5 +++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/eng/common/build.sh b/eng/common/build.sh index 1f8dc49df38..11dcd8942a7 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -42,7 +42,6 @@ while [[ -h "$source" ]]; do done scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" -help=false restore=false build=false rebuild=false diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index aae9c65c1e8..186a0136eda 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -71,6 +71,11 @@ function InitializeDotNetCli([bool]$install) { # Disable first run since we do not need all ASP.NET packages restored. $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 + # Disable telemetry on CI. + if ($ci) { + $env:DOTNET_CLI_TELEMETRY_OPTOUT=1 + } + # Source Build uses DotNetCoreSdkDir variable if ($env:DotNetCoreSdkDir -ne $null) { $env:DOTNET_INSTALL_DIR = $env:DotNetCoreSdkDir diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 42c7c163e5c..374a65bb28d 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -73,6 +73,11 @@ function InitializeDotNetCli { # Disable first run since we want to control all package sources export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 + # Disable telemetry on CI + if [[ $ci == true ]]; then + export DOTNET_CLI_TELEMETRY_OPTOUT=1 + fi + # LTTNG is the logging infrastructure used by Core CLR. Need this variable set # so it doesn't output warnings to the console. export LTTNG_HOME="$HOME" From 5012323f8237144d92d2bf2b095656644a871fbb Mon Sep 17 00:00:00 2001 From: Tomas Matousek Date: Tue, 4 Dec 2018 14:18:57 -0800 Subject: [PATCH 3/9] Misc cleanup --- Build.cmd | 1 - Directory.Build.props | 2 -- Restore.cmd | 3 +-- Test.cmd | 3 +-- eng/Versions.props | 8 +------- 5 files changed, 3 insertions(+), 14 deletions(-) diff --git a/Build.cmd b/Build.cmd index 4afad047142..675fdf83f6a 100644 --- a/Build.cmd +++ b/Build.cmd @@ -1,3 +1,2 @@ @echo off powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\common\Build.ps1""" -restore -build %*" -exit /b %ErrorLevel% diff --git a/Directory.Build.props b/Directory.Build.props index 209cce9a55d..e4bbf4ebb47 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -15,9 +15,7 @@ Tools and packages produced by this repository support infrastructure and are not shipping on NuGet or via any other official channel. --> false - - - 0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7 - - 1.22.0 1.1.2 - - 2.0.0 1.2.0 0.4.0 @@ -69,7 +63,7 @@ $(RestoreSources); https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json; https://dotnet.myget.org/F/symreader-converter/api/v3/index.json; - https:%2F%2Fdotnet.myget.org/F/symreader/api/v3/index.json + https://dotnet.myget.org/F/symreader/api/v3/index.json From d2f55e80e328070220d9300aac577526e41fc00e Mon Sep 17 00:00:00 2001 From: Tomas Matousek Date: Tue, 4 Dec 2018 15:22:32 -0800 Subject: [PATCH 4/9] Update comment --- .../tools/RepoLayout.props | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/RepoLayout.props b/src/Microsoft.DotNet.Arcade.Sdk/tools/RepoLayout.props index 5bb29f4b903..e164c22e2a2 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/RepoLayout.props +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/RepoLayout.props @@ -5,17 +5,6 @@ Properties describing the layout of the repo. --> - - $(MSBuildAllProjects);$(MSBuildThisFileFullPath) From 9655d05e75851b01868f5402189e4fcb06539aec Mon Sep 17 00:00:00 2001 From: Tomas Matousek Date: Thu, 6 Dec 2018 13:08:42 -0800 Subject: [PATCH 5/9] Few more fixes --- eng/common/build.ps1 | 57 ++++++++++++++------------- eng/common/build.sh | 94 +++++++++++++++++++++++--------------------- eng/common/tools.ps1 | 91 +++++++++++++++++++++++++++++++----------- eng/common/tools.sh | 78 ++++++++++++++++++++++++++++-------- 4 files changed, 209 insertions(+), 111 deletions(-) diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 9a1a75e0272..e33cc166a0b 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -4,8 +4,8 @@ Param( [string] $projects = "", [string] $verbosity = "minimal", [string] $msbuildEngine = $null, - [bool] $warnaserror = $true, - [bool] $nodereuse = $true, + [bool] $warnAsError = $true, + [bool] $nodeReuse = $true, [switch] $execute, [switch] $restore, [switch] $deployDeps, @@ -61,19 +61,6 @@ function Print-Usage() { Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)." } -if ($help -or (($properties -ne $null) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) { - Print-Usage - exit 0 -} - -function ConfigureToolset { - # Include custom tools configuration - $script = Join-Path $EngRoot "configure-toolset.ps1" - - if (Test-Path $script) { - . $script - } -} function InitializeCustomToolset { if (-not $restore) { @@ -87,21 +74,10 @@ function InitializeCustomToolset { } } -try { - if ($projects -eq "") { - $projects = Join-Path $RepoRoot "*.sln" - } - - if ($ci) { - $binaryLog = $true - } - - ConfigureToolset +function Build { $toolsetBuildProj = InitializeToolset InitializeCustomToolset - - $buildLog = Join-Path $LogDir "Build.binlog" - $bl = if ($binaryLog) { "/bl:$buildLog" } else { "" } + $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "Build.binlog") } else { "" } MSBuild $toolsetBuildProj ` $bl ` @@ -123,6 +99,31 @@ try { /p:ContinuousIntegrationBuild=$ci ` @properties } + +try { + if ($help -or (($properties -ne $null) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) { + Print-Usage + exit 0 + } + + if ($projects -eq "") { + $projects = Join-Path $RepoRoot "*.sln" + } + + if ($ci) { + $binaryLog = $true + $nodeReuse = $false + } + + # Import custom tools configuration, if present in the repo. + # Note: Import in global scope so that the script set top-level variables without qualification. + $configureToolsetScript = Join-Path $EngRoot "configure-toolset.ps1" + if (Test-Path $configureToolsetScript) { + . $configureToolsetScript + } + + Build +} catch { Write-Host $_ Write-Host $_.Exception diff --git a/eng/common/build.sh b/eng/common/build.sh index 11dcd8942a7..03b4436e1b9 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -54,8 +54,8 @@ sign=false public=false ci=false -warnaserror=true -nodereuse=true +warn_as_error=true +node_reuse=true binary_log=false projects='' @@ -120,11 +120,11 @@ while [[ $# > 0 ]]; do ci=true ;; --warnaserror) - warnaserror=$2 + warn_as_error=$2 shift ;; --nodereuse) - nodereuse=$2 + node_reuse=$2 shift ;; /p:*) @@ -140,15 +140,12 @@ while [[ $# > 0 ]]; do shift done -. "$scriptroot/tools.sh" - -function ConfigureToolset { - local script="$eng_root/configure-toolset.sh" +if [[ "$ci" == true ]]; then + binary_log=true + node_reuse=false +fi - if [[ -a "$script" ]]; then - . "$script" - fi -} +. "$scriptroot/tools.sh" function InitializeCustomToolset { local script="$eng_root/restore-toolset.sh" @@ -158,41 +155,50 @@ function InitializeCustomToolset { fi } -if [[ -z $projects ]]; then - projects="$repo_root/*.sln" -fi +function Build { + InitializeToolset + InitializeCustomToolset -if [[ "$ci" == true ]]; then - $binary_log = true -fi + if [[ -z $projects ]]; then + projects="$repo_root/*.sln" + fi -ConfigureToolset -InitializeToolset -InitializeCustomToolset + local bl="" + if [[ "$binary_log" == true ]]; then + bl="/bl:\"$log_dir/Build.binlog\"" + fi + + MSBuild $_InitializeToolset \ + $bl \ + /p:Configuration=$configuration \ + /p:Projects="$projects" \ + /p:RepoRoot="$repo_root" \ + /p:Restore=$restore \ + /p:Build=$build \ + /p:Rebuild=$rebuild \ + /p:Test=$test \ + /p:Pack=$pack \ + /p:IntegrationTest=$integration_test \ + /p:PerformanceTest=$performance_test \ + /p:Sign=$sign \ + /p:Publish=$publish \ + /p:ContinuousIntegrationBuild=$ci \ + $properties + + ExitWithExitCode 0 +} -build_log="$log_dir/Build.binlog" +# Import custom tools configuration, if present in the repo. +configure_toolset_script="$eng_root/configure-toolset.sh" +if [[ -a "$configure_toolset_script" ]]; then + . "$configure_toolset_script" +fi -if [[ "$binary_log" == true ]]; then - bl="/bl:\"$build_log\"" -else - bl="" +# TODO: https://github.com/dotnet/arcade/issues/1468 +# Temporary workaround to avoid breaking change. +# Remove once repos are updated. +if [[ -n "${useInstalledDotNetCli:-}" ]]; then + use_installed_dotnet_cli="$useInstalledDotNetCli" fi -MSBuild $_InitializeToolset \ - $bl \ - /p:Configuration=$configuration \ - /p:Projects="$projects" \ - /p:RepoRoot="$repo_root" \ - /p:Restore=$restore \ - /p:Build=$build \ - /p:Rebuild=$rebuild \ - /p:Test=$test \ - /p:Pack=$pack \ - /p:IntegrationTest=$integration_test \ - /p:PerformanceTest=$performance_test \ - /p:Sign=$sign \ - /p:Publish=$publish \ - /p:ContinuousIntegrationBuild=$ci \ - $properties - -ExitWithExitCode 0 +Build diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 186a0136eda..0d0ebdc1456 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -1,17 +1,43 @@ -# Initialize variables if they aren't already defined +# Initialize variables if they aren't already defined. +# These may be defined as parameters of the importing script, or set after importing this script. -$ci = if (Test-Path variable:ci) { $ci } else { $false } -$configuration = if (Test-Path variable:configuration) { $configuration } else { "Debug" } -$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci } -$nodereuse = if (Test-Path variable:nodereuse) { $nodereuse } else { !$ci } -$prepareMachine = if (Test-Path variable:prepareMachine) { $prepareMachine } else { $false } -$restore = if (Test-Path variable:restore) { $restore } else { $true } -$verbosity = if (Test-Path variable:verbosity) { $verbosity } else { "minimal" } -$warnaserror = if (Test-Path variable:warnaserror) { $warnaserror } else { $true } -$msbuildEngine = if (Test-Path variable:msbuildEngine) { $msbuildEngine } else { $null } -$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true } +# CI mode - set to true on CI server for PR validation build or official build. +[bool]$ci = if (Test-Path variable:ci) { $ci } else { $false } -$ProcessesToStopOnExit = @("msbuild", "dotnet", "vbcscompiler") +# Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. +[string]$configuration = if (Test-Path variable:configuration) { $configuration } else { "Debug" } + +# Set to true to output binary log from msbuild. Note that emitting binary log slows down the build. +# Binary log must be enabled on CI. +[bool]$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci } + +# Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes). +[bool]$prepareMachine = if (Test-Path variable:prepareMachine) { $prepareMachine } else { $false } + +# True to restore toolsets and dependencies. +[bool]$restore = if (Test-Path variable:restore) { $restore } else { $true } + +# Adjusts msbuild verbosity level. +[string]$verbosity = if (Test-Path variable:verbosity) { $verbosity } else { "minimal" } + +# Set to true to reuse msbuild nodes. Recommended to not reuse on CI. +[bool]$nodeReuse = if (Test-Path variable:nodeReuse) { $nodeReuse } else { !$ci } + +# Configures warning treatment in msbuild. +[bool]$warnAsError = if (Test-Path variable:warnAsError) { $warnAsError } else { $true } + +# Specifies which msbuild engine to use for build: 'vs', 'dotnet' or unspecified (determined based on presence of tools.vs in global.json). +[string]$msbuildEngine = if (Test-Path variable:msbuildEngine) { $msbuildEngine } else { $null } + +# True to attempt using .NET Core already that meets requirements specified in global.json +# installed on the machine instead of downloading one. +[bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true } + +# True to use global NuGet cache instead of restoring packages to repository-local directory. +[bool]$useGlobalNuGetCache = if (Test-Path variable:useGlobalNuGetCache) { $useGlobalNuGetCache } else { !$ci } + +# An array of names of processes to stop on script exit if prepareMachine is true. +$processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @("msbuild", "dotnet", "vbcscompiler") } set-strictmode -version 2.0 $ErrorActionPreference = "Stop" @@ -323,11 +349,27 @@ function GetDefaultMSBuildEngine() { ExitWithExitCode 1 } +function GetNuGetPackageCachePath() { + if ($env:NUGET_PACKAGES -eq $null) { + # Use local cache on CI to ensure deterministic build, + # use global cache in dev builds to avoid cost of downloading packages. + if ($useGlobalNuGetCache) { + $env:NUGET_PACKAGES = Join-Path $env:UserProfile ".nuget\packages" + } else { + $env:NUGET_PACKAGES = Join-Path $RepoRoot ".packages" + } + } + + return $env:NUGET_PACKAGES +} + function InitializeToolset() { if (Test-Path global:_ToolsetBuildProj) { return $global:_ToolsetBuildProj } + $nugetCache = GetNuGetPackageCachePath + $toolsetVersion = $GlobalJson.'msbuild-sdks'.'Microsoft.DotNet.Arcade.Sdk' $toolsetLocationFile = Join-Path $ToolsetDir "$toolsetVersion.txt" @@ -368,7 +410,7 @@ function ExitWithExitCode([int] $exitCode) { function Stop-Processes() { Write-Host "Killing running build processes..." - foreach ($processName in $ProcessesToStopOnExit) { + foreach ($processName in $processesToStopOnExit) { Get-Process -Name $processName -ErrorAction SilentlyContinue | Stop-Process } } @@ -379,11 +421,21 @@ function Stop-Processes() { # Terminates the script if the build fails. # function MSBuild() { + if ($ci) { + if (!$binaryLog) { + throw "Binary log must be enabled in CI build." + } + + if ($nodeReuse) { + throw "Node reuse must be disabled in CI build." + } + } + $buildTool = InitializeBuildTool - $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodereuse" + $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse" - if ($warnaserror) { + if ($warnAsError) { $cmdArgs += " /warnaserror /p:TreatWarningsAsErrors=true" } @@ -433,18 +485,11 @@ $LogDir = Join-Path (Join-Path $ArtifactsDir "log") $configuration $TempDir = Join-Path (Join-Path $ArtifactsDir "tmp") $configuration $GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot "global.json") | ConvertFrom-Json -if ($env:NUGET_PACKAGES -eq $null) { - # Use local cache on CI to ensure deterministic build, - # use global cache in dev builds to avoid cost of downloading packages. - $env:NUGET_PACKAGES = if ($ci) { Join-Path $RepoRoot ".packages" } - else { Join-Path $env:UserProfile ".nuget\packages" } -} - Create-Directory $ToolsetDir +Create-Directory $TempDir Create-Directory $LogDir if ($ci) { - Create-Directory $TempDir $env:TEMP = $TempDir $env:TMP = $TempDir } diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 374a65bb28d..d12c3a0a83c 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -3,19 +3,46 @@ # Stop script if unbound variable found (use ${var:-} if intentional) set -u +# Initialize variables if they aren't already defined. + +# CI mode - set to true on CI server for PR validation build or official build. ci=${ci:-false} + +# Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. configuration=${configuration:-'Debug'} + +# Set to true to output binary log from msbuild. Note that emitting binary log slows down the build. +# Binary log must be enabled on CI. binary_log=${binary_log:-$ci} + +# Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes). prepare_machine=${prepare_machine:-false} + +# True to restore toolsets and dependencies. restore=${restore:-true} + +# Adjusts msbuild verbosity level. verbosity=${verbosity:-'minimal'} -warnaserror=${warnaserror:-true} -useInstalledDotNetCli=${useInstalledDotNetCli:-true} +# Set to true to reuse msbuild nodes. Recommended to not reuse on CI. if [[ "$ci" == true ]]; then - nodereuse=${nodereuse:-false} + node_reuse=${node_reuse:-false} else - nodereuse=${nodereuse:-true} + node_reuse=${node_reuse:-true} +fi + +# Configures warning treatment in msbuild. +warn_as_error=${warn_as_error:-true} + +# True to attempt using .NET Core already that meets requirements specified in global.json +# installed on the machine instead of downloading one. +use_installed_dotnet_cli=${use_installed_dotnet_cli:-true} + +# True to use global NuGet cache instead of restoring packages to repository-local directory. +if [[ "$ci" == true ]]; then + use_global_nuget_cache=${use_global_nuget_cache:-false} +else + use_global_nuget_cache=${use_global_nuget_cache:-true} fi repo_root="$scriptroot/../.." @@ -88,7 +115,7 @@ function InitializeDotNetCli { fi # Find the first path on $PATH that contains the dotnet.exe - if [[ "$useInstalledDotNetCli" == true && -z "${DOTNET_INSTALL_DIR:-}" ]]; then + if [[ "$use_installed_dotnet_cli" == true && -z "${DOTNET_INSTALL_DIR:-}" ]]; then local dotnet_path=`command -v dotnet` if [[ -n "$dotnet_path" ]]; then ResolvePath "$dotnet_path" @@ -171,11 +198,26 @@ function InitializeBuildTool { _InitializeBuildTool="$_InitializeDotNetCli/dotnet" } +function GetNuGetPackageCachePath { + if [[ -z ${NUGET_PACKAGES:-} ]]; then + if [[ "$use_global_nuget_cache" == true ]]; then + export NUGET_PACKAGES="$HOME/.nuget/packages" + else + export NUGET_PACKAGES="$repo_root/.packages" + fi + fi + + # return value + _GetNuGetPackageCachePath=$NUGET_PACKAGES +} + function InitializeToolset { if [[ -n "${_InitializeToolset:-}" ]]; then return fi + GetNuGetPackageCachePath + ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk" local toolset_version=$_ReadGlobalVersion @@ -227,14 +269,26 @@ function StopProcesses { } function MSBuild { + if [[ "$ci" == true ]]; then + if [[ "$binary_log" != true ]]; then + echo "Binary log must be enabled in CI build." >&2 + ExitWithExitCode 1 + fi + + if [[ "$node_reuse" == true ]]; then + echo "Node reuse must be disabled in CI build." >&2 + ExitWithExitCode 1 + fi + fi + InitializeBuildTool local warnaserror_switch="" - if [[ $warnaserror == true ]]; then + if [[ $warn_as_error == true ]]; then warnaserror_switch="/warnaserror" fi - "$_InitializeBuildTool" msbuild /m /nologo /clp:Summary /v:$verbosity /nr:$nodereuse $warnaserror_switch /p:TreatWarningsAsErrors=$warnaserror "$@" + "$_InitializeBuildTool" msbuild /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error "$@" lastexitcode=$? if [[ $lastexitcode != 0 ]]; then @@ -249,19 +303,11 @@ if [[ -z $HOME ]]; then mkdir -p "$HOME" fi -if [[ -z ${NUGET_PACKAGES:-} ]]; then - if [[ $ci == true ]]; then - export NUGET_PACKAGES="$repo_root/.packages" - else - export NUGET_PACKAGES="$HOME/.nuget/packages" - fi -fi - mkdir -p "$toolset_dir" +mkdir -p "$temp_dir" mkdir -p "$log_dir" if [[ $ci == true ]]; then - mkdir -p "$temp_dir" export TEMP="$temp_dir" export TMP="$temp_dir" fi From 6009a508da181fa2f2776f4eee4a0e1176c75930 Mon Sep 17 00:00:00 2001 From: Tomas Matousek Date: Thu, 6 Dec 2018 18:32:39 -0800 Subject: [PATCH 6/9] Fix eng_root --- eng/common/tools.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eng/common/tools.sh b/eng/common/tools.sh index d12c3a0a83c..989da737ddc 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -45,8 +45,10 @@ else use_global_nuget_cache=${use_global_nuget_cache:-true} fi -repo_root="$scriptroot/../.." -eng_root="$scriptroot/.." +ResolvePath "${BASH_SOURCE[0]}" + +eng_root="$_ResolvePath/.." +repo_root="$eng_root/.." artifacts_dir="$repo_root/artifacts" toolset_dir="$artifacts_dir/toolset" log_dir="$artifacts_dir/log/$configuration" From b898daad458fb463fbff3bebe07a05e81208a4c9 Mon Sep 17 00:00:00 2001 From: Tomas Matousek Date: Thu, 6 Dec 2018 18:47:21 -0800 Subject: [PATCH 7/9] Fixes --- eng/common/darc-init.ps1 | 5 +++-- eng/common/darc-init.sh | 13 ++++++++----- eng/common/msbuild.ps1 | 10 +++++----- eng/common/msbuild.sh | 17 ++++++++--------- 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index af182d8f841..9ca150be860 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -3,7 +3,9 @@ $verbosity = "m" function InstallDarcCli { $darcCliPackageName = "microsoft.dotnet.darc" - $dotnet = "$env:DOTNET_INSTALL_DIR\dotnet.exe" + + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" $toolList = Invoke-Expression "& `"$dotnet`" tool list -g" if ($toolList -like "*$darcCliPackageName*") { @@ -17,5 +19,4 @@ function InstallDarcCli { Invoke-Expression "& `"$dotnet`" tool install $darcCliPackageName --version $toolsetVersion -v $verbosity -g" } -InitializeTools InstallDarcCli diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index a0c733a15d4..bad07c3ae61 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -17,10 +17,14 @@ verbosity=m function InstallDarcCli { local darc_cli_package_name="microsoft.dotnet.darc" - local uninstall_command=`$DOTNET_INSTALL_DIR/dotnet tool uninstall $darc_cli_package_name -g` - local tool_list=$($DOTNET_INSTALL_DIR/dotnet tool list -g) + + InitializeDotNetCli + local dotnet_root=$_InitializeDotNetCli + + local uninstall_command=`$dotnet_root/dotnet tool uninstall $darc_cli_package_name -g` + local tool_list=$($dotnet_root/dotnet tool list -g) if [[ $tool_list = *$darc_cli_package_name* ]]; then - echo $($DOTNET_INSTALL_DIR/dotnet tool uninstall $darc_cli_package_name -g) + echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g) fi ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk" @@ -28,8 +32,7 @@ function InstallDarcCli { echo "Installing Darc CLI version $toolset_version..." echo "You may need to restart your command shell if this is the first dotnet tool you have installed." - echo $($DOTNET_INSTALL_DIR/dotnet tool install $darc_cli_package_name --version $toolset_version -v $verbosity -g) + echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $toolset_version -v $verbosity -g) } -InitializeTools InstallDarcCli diff --git a/eng/common/msbuild.ps1 b/eng/common/msbuild.ps1 index 43b837f4df7..3cd667219a6 100644 --- a/eng/common/msbuild.ps1 +++ b/eng/common/msbuild.ps1 @@ -1,8 +1,8 @@ [CmdletBinding(PositionalBinding=$false)] Param( [string] $verbosity = "minimal", - [bool] $warnaserror = $true, - [bool] $nodereuse = $true, + [bool] $warnAsError = $true, + [bool] $nodeReuse = $true, [switch] $ci, [switch] $prepareMachine, [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs @@ -11,13 +11,13 @@ Param( . $PSScriptRoot\tools.ps1 try { - InitializeTools MSBuild @extraArgs - ExitWithExitCode $lastExitCode -} +} catch { Write-Host $_ Write-Host $_.Exception Write-Host $_.ScriptStackTrace ExitWithExitCode 1 } + +ExitWithExitCode 0 \ No newline at end of file diff --git a/eng/common/msbuild.sh b/eng/common/msbuild.sh index b1024487f73..6fe07cb7981 100755 --- a/eng/common/msbuild.sh +++ b/eng/common/msbuild.sh @@ -13,10 +13,10 @@ done scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" verbosity='minimal' -warnaserror=true -nodereuse=true +warn_as_error=true +node_reuse=true prepare_machine=false -extraargs='' +extra_args='' while (($# > 0)); do lowerI="$(echo $1 | awk '{print tolower($0)}')" @@ -26,11 +26,11 @@ while (($# > 0)); do shift 2 ;; --warnaserror) - warnaserror=$2 + warn_as_error=$2 shift 2 ;; --nodereuse) - nodereuse=$2 + node_reuse=$2 shift 2 ;; --ci) @@ -42,7 +42,7 @@ while (($# > 0)); do shift 1 ;; *) - extraargs="$extraargs $1" + extra_args="$extra_args $1" shift 1 ;; esac @@ -50,6 +50,5 @@ done . "$scriptroot/tools.sh" -InitializeTools -MSBuild $extraargs -ExitWithExitCode $? +MSBuild $extra_args +ExitWithExitCode 0 From bd9953e23b391fb55476f4e41e9b17d9212cdf7b Mon Sep 17 00:00:00 2001 From: Tomas Matousek Date: Fri, 7 Dec 2018 12:43:40 -0800 Subject: [PATCH 8/9] fixes --- eng/common/build.ps1 | 20 ++++++++++---------- eng/common/tools.sh | 24 ++++++++++++------------ 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index e33cc166a0b..5241f42e358 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -1,15 +1,15 @@ [CmdletBinding(PositionalBinding=$false)] Param( - [string] $configuration = "Debug", + [string][Alias('c')]$configuration = "Debug", [string] $projects = "", - [string] $verbosity = "minimal", + [string][Alias('v')]$verbosity = "minimal", [string] $msbuildEngine = $null, [bool] $warnAsError = $true, [bool] $nodeReuse = $true, [switch] $execute, - [switch] $restore, + [switch][Alias('r')]$restore, [switch] $deployDeps, - [switch] $build, + [switch][Alias('b')]$build, [switch] $rebuild, [switch] $deploy, [switch] $test, @@ -30,15 +30,15 @@ Param( function Print-Usage() { Write-Host "Common settings:" - Write-Host " -configuration Build configuration Debug, Release" - Write-Host " -verbosity Msbuild verbosity (q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic])" - Write-Host " -binaryLog Output binary log" + Write-Host " -configuration Build configuration: 'Debug' or 'Release' (short: -c)" + Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" + Write-Host " -binaryLog Output binary log (short: -bl)" Write-Host " -help Print help and exit" Write-Host "" Write-Host "Actions:" - Write-Host " -restore Restore dependencies" - Write-Host " -build Build solution" + Write-Host " -restore Restore dependencies (short: -r)" + Write-Host " -build Build solution (short: -b)" Write-Host " -rebuild Rebuild solution" Write-Host " -deploy Deploy built VSIXes" Write-Host " -deployDeps Deploy dependencies (e.g. VSIXes for integration tests)" @@ -48,7 +48,7 @@ function Print-Usage() { Write-Host " -performanceTest Run all performance tests in the solution" Write-Host " -sign Sign build outputs" Write-Host " -publish Publish artifacts (e.g. symbols)" - Write-Host " -publishBuildAssets Push assets to BAR" + Write-Host " -publishBuildAssets Push assets to BAR" Write-Host "" Write-Host "Advanced settings:" diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 989da737ddc..939b2be9d88 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -45,21 +45,10 @@ else use_global_nuget_cache=${use_global_nuget_cache:-true} fi -ResolvePath "${BASH_SOURCE[0]}" - -eng_root="$_ResolvePath/.." -repo_root="$eng_root/.." -artifacts_dir="$repo_root/artifacts" -toolset_dir="$artifacts_dir/toolset" -log_dir="$artifacts_dir/log/$configuration" -temp_dir="$artifacts_dir/tmp/$configuration" - -global_json_file="$repo_root/global.json" - +# Resolve any symlinks in the given path. function ResolvePath { local path=$1 - # resolve $path until the file is no longer a symlink while [[ -h $path ]]; do local dir="$( cd -P "$( dirname "$path" )" && pwd )" path="$(readlink "$path")" @@ -299,6 +288,17 @@ function MSBuild { fi } +ResolvePath "${BASH_SOURCE[0]}" + +eng_root="$( cd -P "$( dirname "$_ResolvePath" )/.." && pwd )" +repo_root="$( cd -P "$( dirname "$_ResolvePath" )/../.." && pwd )" +artifacts_dir="$repo_root/artifacts" +toolset_dir="$artifacts_dir/toolset" +log_dir="$artifacts_dir/log/$configuration" +temp_dir="$artifacts_dir/tmp/$configuration" + +global_json_file="$repo_root/global.json" + # HOME may not be defined in some scenarios, but it is required by NuGet if [[ -z $HOME ]]; then export HOME="$repo_root/artifacts/.home/" From 0ee8eb69a28346eaa252eceb829e4105077b8cd7 Mon Sep 17 00:00:00 2001 From: Tomas Matousek Date: Fri, 7 Dec 2018 12:49:38 -0800 Subject: [PATCH 9/9] tweak --- eng/common/tools.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 939b2be9d88..4d7e304f13d 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -289,9 +289,10 @@ function MSBuild { } ResolvePath "${BASH_SOURCE[0]}" +_script_dir=`dirname "$_ResolvePath"` -eng_root="$( cd -P "$( dirname "$_ResolvePath" )/.." && pwd )" -repo_root="$( cd -P "$( dirname "$_ResolvePath" )/../.." && pwd )" +eng_root=`cd -P "$_script_dir/.." && pwd` +repo_root=`cd -P "$_script_dir/../.." && pwd` artifacts_dir="$repo_root/artifacts" toolset_dir="$artifacts_dir/toolset" log_dir="$artifacts_dir/log/$configuration"