save and publish log from package publishing step

.gitignore

@@ -1,10 +1,9 @@
 # output location
 artifacts/
 
-/tests/scripts/current
+packages/
 
-# auto-generated during the build
-global.json
+/tests/scripts/current
 
 # Patches that may have been generated by scripts.
 # (These aren't generally useful to commit directly; if anything, they should be applied.)

.vsts-signed.yaml

@@ -1,10 +1,17 @@
+variables:
+- name: PB_PublishBlobFeedUrl
+  value: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json
+- group: DotNet-Blob-Feed
+- name: PB_PublishBlobFeedKey
+  value: $(dotnetfeed-storage-access-key-1)
+
 jobs:
 - job: Full_Signed
   pool:
     name: VSEng-MicroBuildVS2017
   timeoutInMinutes: 300
   variables:
-    MSBuildConfiguration: 'Release'
+    BuildConfiguration: 'Release'
   steps:
   # Install Signing Plugin
   - task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@1
@@ -29,15 +36,44 @@ jobs:
     displayName: Publish nightly package to MyGet
     inputs:
       scriptName: 'setup\publish-assets.ps1'
-      arguments: '-binariesPath artifacts\bin -configuration $(MSBuildConfiguration) -branchName $(Build.SourceBranch) -apiKey $(FSharp.MyGetApiKey)'
+      arguments: '-binariesPath artifacts\bin -configuration $(BuildConfiguration) -branchName $(Build.SourceBranch) -apiKey $(FSharp.MyGetApiKey)'
     condition: and(succeeded(), contains(variables['PB_PublishType'], 'myget'))
 
-  # Publish packages to Azure Blob Storage
+  # Package publish
+  - task: CmdLine@1
+    displayName: Restore package publishing
+    inputs:
+      filename: '.nuget\NuGet.exe'
+      arguments: 'restore packages.config -PackagesDirectory packages -Source https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json'
+    condition: and(succeeded(), contains(variables['PB_PublishType'], 'blob'))
   - task: MSBuild@1
     displayName: Publish packages to Azure Blob Storage
     inputs:
       solution: PublishToBlob.proj
-      msbuildArguments: '/t:Build /p:ExpectedFeedUrl=$(PB_PublishBlobFeedUrl) /p:AccountKey=$(PB_PublishBlobFeedKey) /p:ManifestBranch=$(SourceBranch) /p:ManifestCommit=$(SourceVersion) /p:ManifestBuildId=$(OfficialBuildId)'
+      msbuildArguments: '/t:Build /p:Configuration=$(BuildConfiguration) /p:ExpectedFeedUrl=$(PB_PublishBlobFeedUrl) /p:AccountKey=$(PB_PublishBlobFeedKey) /p:ManifestRepouri=$(Build.Repository.Uri) /p:ManifestBranch=$(Build.SourceBranch) /p:ManifestCommit=$(Build.SourceVersion) /p:ManifestBuildId=$(Build.BuildNumber) /bl:$(Build.SourcesDirectory)/artifacts/log/$(BuildConfiguration)/pub/publish.binlog'
+    condition: and(succeeded(), contains(variables['PB_PublishType'], 'blob'))
+  - task: PublishBuildArtifacts@1
+    displayName: Publish publishing bin log
+    inputs:
+      PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(BuildConfiguration)/pub'
+      ArtifactName: 'Publish_bin_log'
+      publishLocation: Container
+    continueOnError: true
+    condition: and(succeeded(), contains(variables['PB_PublishType'], 'blob'))
+  - task: CopyFiles@2
+    displayName: Gather Asset Manifests
+    inputs:
+      SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(BuildConfiguration)/AssetManifest'
+      TargetFolder: '$(Build.StagingDirectory)/AssetManifests'
+    continueOnError: true
+    condition: and(succeeded(), contains(variables['PB_PublishType'], 'blob'))
+  - task: PublishBuildArtifacts@1
+    displayName: Push Asset Manifests
+    inputs:
+      PathtoPublish: '$(Build.StagingDirectory)/AssetManifests'
+      PublishLocation: Container
+      ArtifactName: AssetManifests
+    continueOnError: true
     condition: and(succeeded(), contains(variables['PB_PublishType'], 'blob'))
 
   # Create static drop
@@ -69,7 +105,7 @@ jobs:
   - task: ms-vseng.MicroBuildTasks.4305a8de-ba66-4d8b-b2d1-0dc4ecbbf5e8.MicroBuildUploadVstsDropFolder@1
     displayName: Upload VSTS Drop
     inputs:
-      DropFolder: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(MSBuildConfiguration)\Insertion'
+      DropFolder: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(BuildConfiguration)\Insertion'
     condition: and(succeeded(), contains(variables['PB_PublishType'], 'vsts'))
 
   # Execute cleanup tasks
@@ -94,3 +130,11 @@ jobs:
       sourcePath: '$(DropRoot)\$(Build.DefinitionName)\$(Build.SourceBranchName)\$(Build.BuildNumber)\SymStore'
       usePat: false
     condition: and(succeeded(), contains(variables['PB_PublishType'], 'symweb'))
+
+- template: /eng/common/templates/job/publish-build-assets.yml
+  parameters:
+    dependsOn:
+    - Full_Signed
+    pool:
+      vmImage: vs2017-win2016
+    enablePublishBuildArtifacts: true

Makefile

@@ -8,10 +8,7 @@ all: proto restore build test
 tools:
 	$(CURDIR)/scripts/dotnet-install.sh --version $(DotNetVersion) --install-dir "$(DotNetToolPath)"
 
-global.json: tools
-	echo { \"sdk\": { \"version\": \"$(DotNetVersion)\" } }>global.json
-
-proto: global.json
+proto: tools
 	$(DotNetExe) build-server shutdown
 	$(DotNetExe) restore src/buildtools/buildtools.proj
 	$(DotNetExe) restore src/fsharp/FSharp.Build/FSharp.Build.fsproj
@@ -20,7 +17,7 @@ proto: global.json
 	$(DotNetExe) build src/fsharp/FSharp.Build/FSharp.Build.fsproj -f netstandard2.0 -c Proto
 	$(DotNetExe) build src/fsharp/fsc/fsc.fsproj -f netcoreapp2.1 -c Proto
 
-restore: global.json
+restore:
 	$(DotNetExe) restore src/fsharp/FSharp.Core/FSharp.Core.fsproj
 	$(DotNetExe) restore src/fsharp/FSharp.Build/FSharp.Build.fsproj
 	$(DotNetExe) restore src/fsharp/FSharp.Compiler.Private/FSharp.Compiler.Private.fsproj

NuGet.Config

@@ -3,4 +3,13 @@
   <solution>
     <add key="disableSourceControlIntegration" value="true" />
   </solution>
+  <!-- Only specify feed for Arcade SDK (see https://github.com/Microsoft/msbuild/issues/2982) -->
+  <packageSources>
+    <clear />
+    <add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
+    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
+  </packageSources>
+  <disabledPackageSources>
+    <clear />
+  </disabledPackageSources>
 </configuration>

PublishToBlob.proj

@@ -10,26 +10,33 @@
   <PropertyGroup>
     <FeedTasksPackage>Microsoft.DotNet.Build.Tasks.Feed</FeedTasksPackage>
     <!-- This version should be kept in sync with `packages.config` -->
-    <FeedTasksPackageVersion>2.1.0-prerelease-02419-02</FeedTasksPackageVersion>
+    <FeedTasksPackageVersion>2.2.0-beta.19066.1</FeedTasksPackageVersion>
   </PropertyGroup>
 
   <Import Project="$(MSBuildThisFileDirectory)packages\$(FeedTasksPackage).$(FeedTasksPackageVersion)\build\$(FeedTasksPackage).targets" />
 
   <ItemGroup>
-    <!-- this should pick up both *.nupkg and *.symbols.nupkg -->
-    <NuGetPackages Include="$(MSBuildThisFileDirectory)\artifacts\packages\$(Configuration)\Microsoft.FSharp.Compiler.*.nupkg" />
+    <!-- the string '-rtm-' is important, because that's the package with a per-build unique version number -->
+    <NuGetPackages Include="$(MSBuildThisFileDirectory)artifacts\packages\$(Configuration)\Microsoft.FSharp.Compiler.*-rtm-*.nupkg" />
   </ItemGroup>
 
+  <PropertyGroup>
+    <ArtifactsLogDir>$(MSBuildThisFileDirectory)artifacts\log\$(Configuration)\</ArtifactsLogDir>
+    <Platform Condition="'$(Platform)' == ''">AnyCPU</Platform>
+    <PlatformName Condition="'$(PlatformName)' == ''">$(Platform)</PlatformName>
+    <AssetManifestFilePath>$(ArtifactsLogDir)AssetManifest\$(OS)-$(PlatformName).xml</AssetManifestFilePath>
+  </PropertyGroup>
+
   <Target Name="Build">
     <PushToBlobFeed ExpectedFeedUrl="$(ExpectedFeedUrl)"
                     AccountKey="$(AccountKey)"
                     ItemsToPush="@(NuGetPackages)"
-                    Overwrite="$(PublishOverwrite)"
+                    ManifestBuildData="Location=$(ExpectedFeedUrl)"
+                    ManifestRepoUri="$(ManifestRepoUri)"
                     ManifestBranch="$(ManifestBranch)"
                     ManifestBuildId="$(ManifestBuildId)"
                     ManifestCommit="$(ManifestCommit)"
-                    ManifestName="fsharp"
-                    SkipCreateManifest="false" />
+                    AssetManifestPath="$(AssetManifestFilePath)" />
   </Target>
 
 </Project>

build.cmd

@@ -613,10 +613,6 @@ if not exist %_ngenexe% echo Error: Could not find ngen.exe. && goto :failure
 
 echo ---------------- Done with prepare, starting package restore ----------------
 
-:: create a global.json
-set /p DOTNET_TOOLS_VERSION=<"%~dp0DotnetCLIToolsVersion.txt"
-echo { "sdk": { "version": "%DOTNET_TOOLS_VERSION%" } }>global.json
-
 :: Restore the Tools directory
 call "%~dp0init-tools.cmd"
 set _dotnetexe=%~dp0artifacts\toolset\dotnet\dotnet.exe

eng/common/CIBuild.cmd

@@ -0,0 +1,2 @@
+@echo off
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*"

eng/common/PublishBuildAssets.cmd

@@ -0,0 +1,3 @@
+@echo off
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0sdk-task.ps1""" -msbuildEngine dotnet -restore -projects PublishBuildAssets.proj -ci %*"
+exit /b %ErrorLevel%

eng/common/PublishToPackageFeed.proj

@@ -0,0 +1,37 @@
+<!--
+  This MSBuild file is intended to be used as the body of the default 
+  publishing release pipeline. The release pipeline will use this file
+  to invoke the PushToStaticFeed task that will read the build asset
+  manifest and publish the assets described in the manifest to
+  informed target feeds.
+-->
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>netcoreapp2.1</TargetFramework>
+  </PropertyGroup>
+
+  <Import Project="$(MSBuildThisFileDirectory)MicrosoftDotNetBuildTasksFeedVersion.props" />
+  <Import Project="$(NuGetPackageRoot)microsoft.dotnet.build.tasks.feed\$(MicrosoftDotNetBuildTasksFeedVersion)\build\Microsoft.DotNet.Build.Tasks.Feed.targets" />
+
+  <Target Name="PublishToFeed">
+    <Error Condition="'$(TargetStaticFeed)' == ''" Text="TargetStaticFeed: Target feed for publishing assets wasn't provided." />
+    <Error Condition="'$(AccountKeyToStaticFeed)' == ''" Text="AccountKeyToStaticFeed: Account key for target feed wasn't provided." />
+    <Error Condition="'$(FullPathAssetManifest)' == ''" Text="Full path to asset manifest wasn't provided." />
+    <Error Condition="'$(FullPathBlobBasePath)' == '' AND '$(FullPathPackageBasePath)' == ''" Text="A valid full path to BlobBasePath of PackageBasePath is required." />
+
+    <PushArtifactsInManifestToFeed
+      ExpectedFeedUrl="$(TargetStaticFeed)"
+      AccountKey="$(AccountKeyToStaticFeed)"
+      Overwrite="$(OverrideAssetsWithSameName)"
+      PassIfExistingItemIdentical="$(PassIfExistingItemIdentical)"
+      MaxClients="$(MaxParallelUploads)"
+      UploadTimeoutInMinutes="$(MaxUploadTimeoutInMinutes)"
+      AssetManifestPath="$(FullPathAssetManifest)"
+      BlobAssetsBasePath="$(FullPathBlobBasePath)"
+      PackageAssetsBasePath="$(FullPathPackageBasePath)" />
+  </Target>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.DotNet.Build.Tasks.Feed" Version="$(MicrosoftDotNetBuildTasksFeedVersion)" />
+  </ItemGroup>
+</Project>

eng/common/README.md

@@ -0,0 +1,28 @@
+# Don't touch this folder
+
+                uuuuuuuuuuuuuuuuuuuu
+              u" uuuuuuuuuuuuuuuuuu "u
+            u" u$$$$$$$$$$$$$$$$$$$$u "u
+          u" u$$$$$$$$$$$$$$$$$$$$$$$$u "u
+        u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u
+      u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u
+    u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u
+    $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $
+    $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $
+    $ $$$" ... "$...  ...$" ... "$$$  ... "$$$ $
+    $ $$$u `"$$$$$$$  $$$  $$$$$  $$  $$$  $$$ $
+    $ $$$$$$uu "$$$$  $$$  $$$$$  $$  """ u$$$ $
+    $ $$$""$$$  $$$$  $$$u "$$$" u$$  $$$$$$$$ $
+    $ $$$$....,$$$$$..$$$$$....,$$$$..$$$$$$$$ $
+    $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $
+    "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u"
+      "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u"
+        "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u"
+          "u "$$$$$$$$$$$$$$$$$$$$$$$$" u"
+            "u "$$$$$$$$$$$$$$$$$$$$" u"
+              "u """""""""""""""""" u"
+                """"""""""""""""""""
+
+!!! Changes made in this directory are subject to being overwritten by automation !!!
+
+The files in this directory are shared by all Arcade repos and managed by automation. If you need to make changes to these files, open an issue or submit a pull request to https://github.com/dotnet/arcade first.

eng/common/build.ps1

@@ -0,0 +1,133 @@
+[CmdletBinding(PositionalBinding=$false)]
+Param(
+  [string][Alias('c')]$configuration = "Debug",
+  [string] $projects,
+  [string][Alias('v')]$verbosity = "minimal",
+  [string] $msbuildEngine = $null,
+  [bool] $warnAsError = $true,
+  [bool] $nodeReuse = $true,
+  [switch][Alias('r')]$restore,
+  [switch] $deployDeps,
+  [switch][Alias('b')]$build,
+  [switch] $rebuild,
+  [switch] $deploy,
+  [switch] $test,
+  [switch] $integrationTest,
+  [switch] $performanceTest,
+  [switch] $sign,
+  [switch] $pack,
+  [switch] $publish,
+  [switch][Alias('bl')]$binaryLog,
+  [switch] $ci,
+  [switch] $prepareMachine,
+  [switch] $help,
+  [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
+)
+
+. $PSScriptRoot\tools.ps1
+
+function Print-Usage() {
+    Write-Host "Common settings:"
+    Write-Host "  -configuration <value>  Build configuration: 'Debug' or 'Release' (short: -c)"
+    Write-Host "  -verbosity <value>      Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)"
+    Write-Host "  -binaryLog              Output binary log (short: -bl)"
+    Write-Host "  -help                   Print help and exit"
+    Write-Host ""
+
+    Write-Host "Actions:"
+    Write-Host "  -restore                Restore dependencies (short: -r)"
+    Write-Host "  -build                  Build solution (short: -b)"
+    Write-Host "  -rebuild                Rebuild solution"
+    Write-Host "  -deploy                 Deploy built VSIXes"
+    Write-Host "  -deployDeps             Deploy dependencies (e.g. VSIXes for integration tests)"
+    Write-Host "  -test                   Run all unit tests in the solution"
+    Write-Host "  -pack                   Package build outputs into NuGet packages and Willow components"
+    Write-Host "  -integrationTest        Run all integration tests in the solution"
+    Write-Host "  -performanceTest        Run all performance tests in the solution"
+    Write-Host "  -sign                   Sign build outputs"
+    Write-Host "  -publish                Publish artifacts (e.g. symbols)"
+    Write-Host ""
+
+    Write-Host "Advanced settings:"
+    Write-Host "  -projects <value>       Semi-colon delimited list of sln/proj's to build. Globbing is supported (*.sln)"
+    Write-Host "  -ci                     Set when running on CI server"
+    Write-Host "  -prepareMachine         Prepare machine for CI run"
+    Write-Host "  -msbuildEngine <value>  Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
+    Write-Host ""
+    Write-Host "Command line arguments not listed above are passed thru to msbuild."
+    Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)."
+}
+
+function InitializeCustomToolset {
+  if (-not $restore) {
+    return
+  }
+
+  $script = Join-Path $EngRoot "restore-toolset.ps1"
+
+  if (Test-Path $script) {
+    . $script
+  }
+}
+
+function Build {
+  $toolsetBuildProj = InitializeToolset
+  InitializeCustomToolset
+
+  $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "Build.binlog") } else { "" }
+
+  if ($projects) {
+    # Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons.
+    # Explicitly set the type as string[] because otherwise PowerShell would make this char[] if $properties is empty.
+    [string[]] $msbuildArgs = $properties
+    $msbuildArgs += "/p:Projects=$projects"
+    $properties = $msbuildArgs
+  }
+
+  MSBuild $toolsetBuildProj `
+    $bl `
+    /p:Configuration=$configuration `
+    /p:RepoRoot=$RepoRoot `
+    /p:Restore=$restore `
+    /p:DeployDeps=$deployDeps `
+    /p:Build=$build `
+    /p:Rebuild=$rebuild `
+    /p:Deploy=$deploy `
+    /p:Test=$test `
+    /p:Pack=$pack `
+    /p:IntegrationTest=$integrationTest `
+    /p:PerformanceTest=$performanceTest `
+    /p:Sign=$sign `
+    /p:Publish=$publish `
+    /p:ContinuousIntegrationBuild=$ci `
+    @properties
+}
+
+try {
+  if ($help -or (($properties -ne $null) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) {
+    Print-Usage
+    exit 0
+  }
+
+  if ($ci) {
+    $binaryLog = $true
+    $nodeReuse = $false
+  }
+
+  # Import custom tools configuration, if present in the repo.
+  # Note: Import in global scope so that the script set top-level variables without qualification.
+  $configureToolsetScript = Join-Path $EngRoot "configure-toolset.ps1"
+  if (Test-Path $configureToolsetScript) {
+    . $configureToolsetScript
+  }
+
+  Build
+}
+catch {
+  Write-Host $_
+  Write-Host $_.Exception
+  Write-Host $_.ScriptStackTrace
+  ExitWithExitCode 1
+}
+
+ExitWithExitCode 0