Add build manifest

.gitignore

@@ -3,9 +3,9 @@ syntax: glob
 ### VisualStudio ###
 
 # Tools directory
-/[Tt]ools/
 .dotnet/
 .packages/
+.tools/
 
 # User-specific files
 *.suo

Arcade.sln

@@ -95,7 +95,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DotNet.Helix.JobS
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DotNet.Build.Tasks.TargetFramework.Sdk", "src\Microsoft.DotNet.Build.Tasks.TargetFramework.Sdk\src\Microsoft.DotNet.Build.Tasks.TargetFramework.Sdk.csproj", "{E83B25A9-66C3-4E15-9BC3-E843CC471622}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.DotNet.Helix.Sdk.Tests", "src\Microsoft.DotNet.Helix\Sdk.Tests\Microsoft.DotNet.Helix.Sdk.Tests\Microsoft.DotNet.Helix.Sdk.Tests.csproj", "{03390E61-9DC1-4893-93A4-193D76C16034}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DotNet.Helix.Sdk.Tests", "src\Microsoft.DotNet.Helix\Sdk.Tests\Microsoft.DotNet.Helix.Sdk.Tests\Microsoft.DotNet.Helix.Sdk.Tests.csproj", "{03390E61-9DC1-4893-93A4-193D76C16034}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DotNet.AzureDevOps.Build", "src\Microsoft.DotNet.AzureDevOps.Build\Microsoft.DotNet.AzureDevOps.Build.csproj", "{107F83D6-3214-4713-898D-373FCF051366}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -607,6 +609,18 @@ Global
 		{03390E61-9DC1-4893-93A4-193D76C16034}.Release|x64.Build.0 = Release|Any CPU
 		{03390E61-9DC1-4893-93A4-193D76C16034}.Release|x86.ActiveCfg = Release|Any CPU
 		{03390E61-9DC1-4893-93A4-193D76C16034}.Release|x86.Build.0 = Release|Any CPU
+		{107F83D6-3214-4713-898D-373FCF051366}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{107F83D6-3214-4713-898D-373FCF051366}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{107F83D6-3214-4713-898D-373FCF051366}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{107F83D6-3214-4713-898D-373FCF051366}.Debug|x64.Build.0 = Debug|Any CPU
+		{107F83D6-3214-4713-898D-373FCF051366}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{107F83D6-3214-4713-898D-373FCF051366}.Debug|x86.Build.0 = Debug|Any CPU
+		{107F83D6-3214-4713-898D-373FCF051366}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{107F83D6-3214-4713-898D-373FCF051366}.Release|Any CPU.Build.0 = Release|Any CPU
+		{107F83D6-3214-4713-898D-373FCF051366}.Release|x64.ActiveCfg = Release|Any CPU
+		{107F83D6-3214-4713-898D-373FCF051366}.Release|x64.Build.0 = Release|Any CPU
+		{107F83D6-3214-4713-898D-373FCF051366}.Release|x86.ActiveCfg = Release|Any CPU
+		{107F83D6-3214-4713-898D-373FCF051366}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

azure-pipelines.yml

@@ -33,6 +33,7 @@ stages:
           manifests: true
       enableMicrobuild: true
       enablePublishUsingPipelines: true
+      useBuildManifest: ${{ variables['_UseBuildManifest'] }}
       workspace:
         clean: all
       jobs:
@@ -192,3 +193,4 @@ stages:
         -TsaRepositoryName "Arcade"
         -TsaCodebaseName "Arcade"
         -TsaPublish $True'
+      useBuildManifest: ${{ variables['_UseBuildManifest'] }}

eng/Versions.props

@@ -38,7 +38,7 @@
     <MonoOptionsVersion>5.3.0.1</MonoOptionsVersion>
     <McMasterExtensionsCommandLineUtils>2.3.0</McMasterExtensionsCommandLineUtils>
     <NewtonsoftJsonVersion>9.0.1</NewtonsoftJsonVersion>
-    <MicrosoftBclJsonSourcesVersion>4.6.0-preview4.19202.2</MicrosoftBclJsonSourcesVersion>
+    <SystemTextJsonVersion>4.7.0</SystemTextJsonVersion>
     <NuGetVersioningVersion>4.4.0</NuGetVersioningVersion>
     <NuGetVersion>5.3.0</NuGetVersion>
     <OctokitVersion>0.32.0</OctokitVersion>
@@ -50,12 +50,12 @@
     <SystemIOCompressionVersion>4.3.0</SystemIOCompressionVersion>
     <SystemIOPackagingVersion>4.5.0</SystemIOPackagingVersion>
     <SystemIOFileSystemPrimitivesVersion>4.3.0</SystemIOFileSystemPrimitivesVersion>
-    <SystemMemoryVersion>4.5.1</SystemMemoryVersion>
-    <SystemNumericsVectorsVersion>4.4.0</SystemNumericsVectorsVersion>
+    <SystemMemoryVersion>4.5.3</SystemMemoryVersion>
+    <SystemNumericsVectorsVersion>4.5.0</SystemNumericsVectorsVersion>
     <SystemReflectionMetadataVersion>1.4.2</SystemReflectionMetadataVersion>
-    <SystemRuntimeCompilerServicesUnsafeVersion>4.5.0</SystemRuntimeCompilerServicesUnsafeVersion>
+    <SystemRuntimeCompilerServicesUnsafeVersion>4.7.0</SystemRuntimeCompilerServicesUnsafeVersion>
     <SystemTextEncodingsWebVersion>4.5.0</SystemTextEncodingsWebVersion>
-    <SystemThreadingTasksExtensionVersion>4.5.1</SystemThreadingTasksExtensionVersion>
+    <SystemThreadingTasksExtensionVersion>4.5.2</SystemThreadingTasksExtensionVersion>
     <SystemValueTupleVersion>4.4.0</SystemValueTupleVersion>
     <WindowsAzureStorageVersion>8.5.0</WindowsAzureStorageVersion>
     <XUnitVersion>2.4.1</XUnitVersion>

eng/common-variables.yml

@@ -10,6 +10,8 @@ variables:
     value: False
   - name: _InternalBuildArgs
     value: ''
+  - name: _UseBuildManifest
+    value: False
 
   - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
     - name: _RunAsPublic
@@ -18,6 +20,8 @@ variables:
       value: True
     - name: _SignType
       value: real
+    - name: _UseBuildManifest
+      value: False
     # DotNet-Blob-Feed provides: dotnetfeed-storage-access-key-1
     # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT
     # DotNet-HelixApi-Access provides: HelixApiAccessToken

eng/common/sdk-task.ps1

@@ -57,6 +57,18 @@ try {
     ExitWithExitCode 1
   }
 
+  if( $msbuildEngine -eq "vs") {
+    # Ensure desktop MSBuild is available for sdk tasks.
+    if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "vs" )) {
+      $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.4`" }") -MemberType NoteProperty
+    }
+    if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
+      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.4.0-alpha" -MemberType NoteProperty
+    }
+
+    InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true
+  }
+
   $taskProject = GetSdkTaskProject $task
   if (!(Test-Path $taskProject)) {
     Write-PipelineTelemetryError -Category 'Build' -Message "Unknown task: $task" -ForegroundColor Red

eng/common/templates/job/job.yml

@@ -24,6 +24,7 @@ parameters:
   enablePublishBuildAssets: false
   enablePublishTestResults: false
   enablePublishUsingPipelines: false
+  useBuildManifest: false
   mergeTestResults: false
   testRunTitle: $(AgentOsName)-$(BuildConfiguration)-xunit
   name: ''
@@ -218,3 +219,12 @@ jobs:
         ArtifactName: AssetManifests
       continueOnError: ${{ parameters.continueOnError }}
       condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
+
+  - ${{ if eq(parameters.useBuildManifest, true) }}:
+    - task: PublishBuildArtifacts@1
+      displayName: Publish Build Manifest
+      inputs:
+        PathToPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/manifest.props'
+        PublishLocation: Container
+        ArtifactName: BuildManifests
+      continueOnError: ${{ parameters.continueOnError }}

eng/common/templates/post-build/post-build.yml

@@ -15,6 +15,7 @@ parameters:
   symbolPublishingAdditionalParameters: ''
   artifactsPublishingAdditionalParameters: ''
   signingValidationAdditionalParameters: ''
+  useBuildManifest: false
 
   # Which stages should finish execution before post-build stages start
   validateDependsOn:
@@ -113,6 +114,16 @@ stages:
     pool:
       vmImage: 'windows-2019'
     steps:
+      - ${{ if eq(parameters.useBuildManifest, true) }}:
+        - task: DownloadBuildArtifacts@0
+          displayName: Download build manifest
+          inputs:
+            buildType: specific
+            buildVersionToDownload: specific
+            project: $(AzDOProjectName)
+            pipeline: $(AzDOPipelineId)
+            buildId: $(AzDOBuildId)
+            artifactName: BuildManifests
       - task: DownloadBuildArtifacts@0
         displayName: Download Package Artifacts
         inputs:
@@ -135,11 +146,13 @@ stages:
           filePath: eng\common\enable-cross-org-publishing.ps1
           arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
 
+      # Signing validation will optionally work with the buildmanifest file which is downloaded from
+      # Azure DevOps above.
       - task: PowerShell@2
         displayName: Validate
         inputs:
           filePath: eng\common\sdk-task.ps1
-          arguments: -task SigningValidation -restore -msbuildEngine dotnet
+          arguments: -task SigningValidation -restore -msbuildEngine vs
             /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts'
             /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt'
             ${{ parameters.signingValidationAdditionalParameters }}

src/Microsoft.DotNet.Arcade.Sdk/Microsoft.DotNet.Arcade.Sdk.csproj

@@ -12,7 +12,7 @@
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>
 
     <Description>Common toolset for repositories</Description>
-    <PackageTags>Roslyn Build Repository Toolset MSBuild SDK</PackageTags>
+    <PackageTags>Arcade Build Repository Toolset MSBuild SDK</PackageTags>
     <DevelopmentDependency>true</DevelopmentDependency>
     <PackageType>MSBuildSdk</PackageType>
 
@@ -33,7 +33,7 @@
 
   <!-- Required for compiling "InstallDotNetCore.cs" -->
   <ItemGroup Condition="'$(DotNetBuildFromSource)' != 'true'">
-    <PackageReference Include="Microsoft.Bcl.Json.Sources" Version="$(MicrosoftBclJsonSourcesVersion)" PrivateAssets="All" />
+    <PackageReference Include="System.Text.Json" Version="$(SystemTextJsonVersion)" PrivateAssets="All" />
     <PackageReference Include="System.Buffers" Version="$(SystemBuffersVersion)" />
     <PackageReference Include="System.Memory" Version="$(SystemMemoryVersion)" />
     <PackageReference Include="System.Numerics.Vectors" Version="$(SystemNumericsVectorsVersion)" />

src/Microsoft.DotNet.Arcade.Sdk/tools/Publish.proj

@@ -39,7 +39,7 @@
     <PublishDependsOnTargets Condition="$(PublishToSymbolServer)">$(PublishDependsOnTargets);PublishSymbols</PublishDependsOnTargets>
     <PublishDependsOnTargets Condition="$(PublishToSourceBuildStorage)">$(PublishDependsOnTargets);PublishToSourceBuildStorage</PublishDependsOnTargets>
     <PublishDependsOnTargets Condition="$(DotNetPublishUsingPipelines)">$(PublishDependsOnTargets);PublishToAzureDevOpsArtifacts</PublishDependsOnTargets>
-    <PublishDependsOnTargets>BeforePublish;$(PublishDependsOnTargets)</PublishDependsOnTargets>
+    <PublishDependsOnTargets>BeforePublish;GenerateBuildManifest;$(PublishDependsOnTargets)</PublishDependsOnTargets>
   </PropertyGroup>
 
   <Import Project="$(NuGetPackageRoot)microsoft.dotnet.build.tasks.feed\$(MicrosoftDotNetBuildTasksFeedVersion)\build\Microsoft.DotNet.Build.Tasks.Feed.targets"/>
@@ -238,4 +238,24 @@
                     Condition="$(PublishToSymbolServer)"/>
   </Target>
 
+  <PropertyGroup>
+    <AzureDevOpsBuildTasksAssembly Condition="'$(MSBuildRuntimeType)' != 'Core'">$(NuGetPackageRoot)Microsoft.DotNet.AzureDevOps.Build\$(ArcadeSdkVersion)\tools\net472\Microsoft.DotNet.AzureDevOps.Build.dll</AzureDevOpsBuildTasksAssembly>
+    <AzureDevOpsBuildTasksAssembly Condition="'$(MSBuildRuntimeType)' == 'Core'">$(NuGetPackageRoot)Microsoft.DotNet.AzureDevOps.Build\$(ArcadeSdkVersion)\tools\netcoreapp2.1\Microsoft.DotNet.AzureDevOps.Build.dll</AzureDevOpsBuildTasksAssembly>
+  </PropertyGroup>
+
+  <Import Project="Sign.props" />
+  <UsingTask TaskName="Microsoft.DotNet.AzureDevOps.Build.GenerateAzureDevOpsBuildManifest" AssemblyFile="$(AzureDevOpsBuildTasksAssembly)"/>
+
+  <Target Name="GenerateBuildManifest">
+    <GenerateAzureDevOpsBuildManifest
+      AzureDevOpsCollectionUri="$(SYSTEM_TEAMFOUNDATIONCOLLECTIONURI)"
+      AzureDevOpsProject="$(SYSTEM_TEAMPROJECT)"
+      AzureDevOpsBuildId="$(BUILD_BUILDID)"
+      ItemsToSign="@(ItemsToSign)"
+      StrongNameSignInfo="@(StrongNameSignInfo)"
+      FileSignInfo="@(FileSignInfo)"
+      FileExtensionSignInfo="@(FileExtensionSignInfo)"
+      ManifestPath="$(ArtifactsLogDir)/manifest.props" />
+  </Target>
+
 </Project>

src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/SigningValidation.proj

@@ -14,45 +14,47 @@
       - EnableStrongNameCheck   : Whether strong name check should be performed.
   -->
 
+  <PropertyGroup>
+    <BuildManifestFile Condition="Exists('$(BUILD_ARTIFACTSTAGINGDIRECTORY)/BuildManifests/manifest.props')">$(BUILD_ARTIFACTSTAGINGDIRECTORY)/BuildManifests/manifest.props</BuildManifestFile>
+  </PropertyGroup>
+
+  <Import Condition="'$(BuildManifestFile)' == ''" Project="$(RepositoryEngineeringDir)Signing.props" />
+  <Import Condition="Exists('$(BuildManifestFile)')" Project="$(BuildManifestFile)" />
+
   <PropertyGroup>
     <TargetFramework>netcoreapp2.1</TargetFramework>
     <NETCORE_ENGINEERING_TELEMETRY>Build</NETCORE_ENGINEERING_TELEMETRY>
+    <SignCheckTaskAssembly>$(NuGetPackageRoot)Microsoft.DotNet.SignCheck\$(MicrosoftDotNetSignCheckVersion)\tools\Microsoft.DotNet.SignCheck.exe</SignCheckTaskAssembly>
   </PropertyGroup>
 
+  <UsingTask TaskName="SignCheck.SignCheckTask" AssemblyFile="$(SignCheckTaskAssembly)" />
+
   <Target Name="Execute">
     <PropertyGroup>
-      <SignCheckToolPath>$(NuGetPackageRoot)Microsoft.DotNet.SignCheck\$(MicrosoftDotNetSignCheckVersion)\tools\Microsoft.DotNet.SignCheck.exe</SignCheckToolPath>
-
       <SignCheckInputDir>$(PackageBasePath)</SignCheckInputDir>
       <SignCheckLog Condition="'$(SignCheckLog)' == ''">$(ArtifactsLogDir)\signcheck.log</SignCheckLog>
       <SignCheckErrorLog Condition="'$(SignCheckErrorLog)' == ''">$(ArtifactsLogDir)\signcheck.errors.log</SignCheckErrorLog>
+      <InputFiles Condition="'$(BuildManifestFile)' == ''">$(SignCheckInputDir)</InputFiles>
+      <InputFiles Condition="'$(BuildManifestFile)' != ''">@(ItemsToSign)</InputFiles>
     </PropertyGroup>
-
-    <ItemGroup>
+
+    <Error Condition="'$(MSBuildRuntimeType)' == 'Core'" Text="Signing validation task does not run on core." />
+
+    <Message Text="Using build manifest file '$(BuildManifestFile)'" Condition="Exists('$(BuildManifestFile)')" />
       <!--
         Documentation for these arguments is available here:
         https://github.com/dotnet/arcade/tree/master/src/SignCheck
       -->
-      <SignCheckArgs Include="--recursive" />
-      <SignCheckArgs Include="--traverse-subfolders" />
-      <SignCheckArgs Include="--file-status AllFiles" />
-      <SignCheckArgs Include="--log-file $(SignCheckLog)" />
-      <SignCheckArgs Include="--error-log-file $(SignCheckErrorLog)" />
-      <SignCheckArgs Include="--input-files &quot;$(SignCheckInputDir)&quot;" />
-
-      <SignCheckArgs Include="--exclusions-file &quot;$(SignCheckExclusionsFile)&quot;" Condition="'$(SignCheckExclusionsFile)' != '' and Exists($(SignCheckExclusionsFile))" />
-      <SignCheckArgs Include="--verify-jar" Condition="'$(EnableJarSigningCheck)' == 'true'" />
-      <SignCheckArgs Include="--verify-strongname" Condition="'$(EnableStrongNameCheck)' == 'true'" />
-    </ItemGroup>
-
-    <!--
-      IgnoreExitCode='true' because the tool doesn't return '0' on success.
-    -->
-    <Exec 
-      Command="&quot;$(SignCheckToolPath)&quot; @(SignCheckArgs, ' ')"
-      IgnoreExitCode='true' 
-      ConsoleToMsBuild="false" 
-      StandardErrorImportance="high" />
+    <SignCheckTask 
+      Recursive="true"
+      FileStatus="AllFiles"
+      InputFiles="$(InputFiles)"
+      ExclusionsFile="$(SignCheckExclusionsFile)"
+      EnableJarSignatureVerification="$(EnableJarSigningCheck)"
+      VerifyStrongName="$(EnableStrongNameCheck)"
+      LogFile="$(SignCheckLog)"
+      ErrorLogFile="$(SignCheckErrorLog)"
+      ArtifactFolder="$(SignCheckInputDir)"/>
 
     <Error 
       Text="Signing validation failed. Check $(SignCheckErrorLog) for more information." 
@@ -61,7 +63,6 @@
     <Message
       Text="##vso[artifact.upload containerfolder=LogFiles;artifactname=LogFiles]{SignCheckErrorLog}"
       Condition="Exists($(SignCheckErrorLog)) and '$([System.IO.File]::ReadAllText($(SignCheckErrorLog)))' != ''" />
-
   </Target>
 
   <ItemGroup>