diff --git a/.gitignore b/.gitignore index 5e16dcab58e..19c954ea8f3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,9 +3,9 @@ syntax: glob ### VisualStudio ### # Tools directory -/[Tt]ools/ .dotnet/ .packages/ +.tools/ # User-specific files *.suo diff --git a/Arcade.sln b/Arcade.sln index dc123c418b4..f43a6e06d11 100644 --- a/Arcade.sln +++ b/Arcade.sln @@ -95,7 +95,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DotNet.Helix.JobS EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DotNet.Build.Tasks.TargetFramework.Sdk", "src\Microsoft.DotNet.Build.Tasks.TargetFramework.Sdk\src\Microsoft.DotNet.Build.Tasks.TargetFramework.Sdk.csproj", "{E83B25A9-66C3-4E15-9BC3-E843CC471622}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.DotNet.Helix.Sdk.Tests", "src\Microsoft.DotNet.Helix\Sdk.Tests\Microsoft.DotNet.Helix.Sdk.Tests\Microsoft.DotNet.Helix.Sdk.Tests.csproj", "{03390E61-9DC1-4893-93A4-193D76C16034}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DotNet.Helix.Sdk.Tests", "src\Microsoft.DotNet.Helix\Sdk.Tests\Microsoft.DotNet.Helix.Sdk.Tests\Microsoft.DotNet.Helix.Sdk.Tests.csproj", "{03390E61-9DC1-4893-93A4-193D76C16034}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DotNet.AzureDevOps.Build", "src\Microsoft.DotNet.AzureDevOps.Build\Microsoft.DotNet.AzureDevOps.Build.csproj", "{107F83D6-3214-4713-898D-373FCF051366}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -607,6 +609,18 @@ Global {03390E61-9DC1-4893-93A4-193D76C16034}.Release|x64.Build.0 = Release|Any CPU {03390E61-9DC1-4893-93A4-193D76C16034}.Release|x86.ActiveCfg = Release|Any CPU {03390E61-9DC1-4893-93A4-193D76C16034}.Release|x86.Build.0 = Release|Any CPU + {107F83D6-3214-4713-898D-373FCF051366}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {107F83D6-3214-4713-898D-373FCF051366}.Debug|Any CPU.Build.0 = Debug|Any CPU + {107F83D6-3214-4713-898D-373FCF051366}.Debug|x64.ActiveCfg = Debug|Any CPU + {107F83D6-3214-4713-898D-373FCF051366}.Debug|x64.Build.0 = Debug|Any CPU + {107F83D6-3214-4713-898D-373FCF051366}.Debug|x86.ActiveCfg = Debug|Any CPU + {107F83D6-3214-4713-898D-373FCF051366}.Debug|x86.Build.0 = Debug|Any CPU + {107F83D6-3214-4713-898D-373FCF051366}.Release|Any CPU.ActiveCfg = Release|Any CPU + {107F83D6-3214-4713-898D-373FCF051366}.Release|Any CPU.Build.0 = Release|Any CPU + {107F83D6-3214-4713-898D-373FCF051366}.Release|x64.ActiveCfg = Release|Any CPU + {107F83D6-3214-4713-898D-373FCF051366}.Release|x64.Build.0 = Release|Any CPU + {107F83D6-3214-4713-898D-373FCF051366}.Release|x86.ActiveCfg = Release|Any CPU + {107F83D6-3214-4713-898D-373FCF051366}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c2b8ee93afa..2a25fd4dd63 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -33,6 +33,7 @@ stages: manifests: true enableMicrobuild: true enablePublishUsingPipelines: true + useBuildManifest: ${{ variables['_UseBuildManifest'] }} workspace: clean: all jobs: @@ -192,3 +193,4 @@ stages: -TsaRepositoryName "Arcade" -TsaCodebaseName "Arcade" -TsaPublish $True' + useBuildManifest: ${{ variables['_UseBuildManifest'] }} \ No newline at end of file diff --git a/eng/Versions.props b/eng/Versions.props index 3a8d3079d21..e5be6b06885 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -38,7 +38,7 @@ 5.3.0.1 2.3.0 9.0.1 - 4.6.0-preview4.19202.2 + 4.7.0 4.4.0 5.3.0 0.32.0 @@ -50,12 +50,12 @@ 4.3.0 4.5.0 4.3.0 - 4.5.1 - 4.4.0 + 4.5.3 + 4.5.0 1.4.2 - 4.5.0 + 4.7.0 4.5.0 - 4.5.1 + 4.5.2 4.4.0 8.5.0 2.4.1 diff --git a/eng/common-variables.yml b/eng/common-variables.yml index bd9386ca1f6..17d057ba3f5 100644 --- a/eng/common-variables.yml +++ b/eng/common-variables.yml @@ -10,6 +10,8 @@ variables: value: False - name: _InternalBuildArgs value: '' + - name: _UseBuildManifest + value: False - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - name: _RunAsPublic @@ -18,6 +20,8 @@ variables: value: True - name: _SignType value: real + - name: _UseBuildManifest + value: False # DotNet-Blob-Feed provides: dotnetfeed-storage-access-key-1 # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT # DotNet-HelixApi-Access provides: HelixApiAccessToken diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 3872af59b97..79c25e7f3ef 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -57,6 +57,18 @@ try { ExitWithExitCode 1 } + if( $msbuildEngine -eq "vs") { + # Ensure desktop MSBuild is available for sdk tasks. + if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "vs" )) { + $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.4`" }") -MemberType NoteProperty + } + if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.4.0-alpha" -MemberType NoteProperty + } + + InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true + } + $taskProject = GetSdkTaskProject $task if (!(Test-Path $taskProject)) { Write-PipelineTelemetryError -Category 'Build' -Message "Unknown task: $task" -ForegroundColor Red diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 536c15c4641..fc39647f4b1 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -24,6 +24,7 @@ parameters: enablePublishBuildAssets: false enablePublishTestResults: false enablePublishUsingPipelines: false + useBuildManifest: false mergeTestResults: false testRunTitle: $(AgentOsName)-$(BuildConfiguration)-xunit name: '' @@ -218,3 +219,12 @@ jobs: ArtifactName: AssetManifests continueOnError: ${{ parameters.continueOnError }} condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) + + - ${{ if eq(parameters.useBuildManifest, true) }}: + - task: PublishBuildArtifacts@1 + displayName: Publish Build Manifest + inputs: + PathToPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/manifest.props' + PublishLocation: Container + ArtifactName: BuildManifests + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index fbab4cb5dce..957bf3018b7 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -15,6 +15,7 @@ parameters: symbolPublishingAdditionalParameters: '' artifactsPublishingAdditionalParameters: '' signingValidationAdditionalParameters: '' + useBuildManifest: false # Which stages should finish execution before post-build stages start validateDependsOn: @@ -113,6 +114,16 @@ stages: pool: vmImage: 'windows-2019' steps: + - ${{ if eq(parameters.useBuildManifest, true) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download build manifest + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BuildManifests - task: DownloadBuildArtifacts@0 displayName: Download Package Artifacts inputs: @@ -135,11 +146,13 @@ stages: filePath: eng\common\enable-cross-org-publishing.ps1 arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. - task: PowerShell@2 displayName: Validate inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine dotnet + arguments: -task SigningValidation -restore -msbuildEngine vs /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' ${{ parameters.signingValidationAdditionalParameters }} diff --git a/src/Microsoft.DotNet.Arcade.Sdk/Microsoft.DotNet.Arcade.Sdk.csproj b/src/Microsoft.DotNet.Arcade.Sdk/Microsoft.DotNet.Arcade.Sdk.csproj index 50ac43548a6..1cd126fe63b 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/Microsoft.DotNet.Arcade.Sdk.csproj +++ b/src/Microsoft.DotNet.Arcade.Sdk/Microsoft.DotNet.Arcade.Sdk.csproj @@ -12,7 +12,7 @@ true Common toolset for repositories - Roslyn Build Repository Toolset MSBuild SDK + Arcade Build Repository Toolset MSBuild SDK true MSBuildSdk @@ -33,7 +33,7 @@ - + diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/Publish.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/Publish.proj index ff87aefcf04..a3e82238832 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/Publish.proj +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/Publish.proj @@ -39,7 +39,7 @@ $(PublishDependsOnTargets);PublishSymbols $(PublishDependsOnTargets);PublishToSourceBuildStorage $(PublishDependsOnTargets);PublishToAzureDevOpsArtifacts - BeforePublish;$(PublishDependsOnTargets) + BeforePublish;GenerateBuildManifest;$(PublishDependsOnTargets) @@ -238,4 +238,24 @@ Condition="$(PublishToSymbolServer)"/> + + $(NuGetPackageRoot)Microsoft.DotNet.AzureDevOps.Build\$(ArcadeSdkVersion)\tools\net472\Microsoft.DotNet.AzureDevOps.Build.dll + $(NuGetPackageRoot)Microsoft.DotNet.AzureDevOps.Build\$(ArcadeSdkVersion)\tools\netcoreapp2.1\Microsoft.DotNet.AzureDevOps.Build.dll + + + + + + + + + diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/SigningValidation.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/SigningValidation.proj index e86f22ad477..d0f4f0d295d 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/SigningValidation.proj +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/SigningValidation.proj @@ -14,45 +14,47 @@ - EnableStrongNameCheck : Whether strong name check should be performed. --> + + $(BUILD_ARTIFACTSTAGINGDIRECTORY)/BuildManifests/manifest.props + + + + + netcoreapp2.1 Build + $(NuGetPackageRoot)Microsoft.DotNet.SignCheck\$(MicrosoftDotNetSignCheckVersion)\tools\Microsoft.DotNet.SignCheck.exe + + - $(NuGetPackageRoot)Microsoft.DotNet.SignCheck\$(MicrosoftDotNetSignCheckVersion)\tools\Microsoft.DotNet.SignCheck.exe - $(PackageBasePath) $(ArtifactsLogDir)\signcheck.log $(ArtifactsLogDir)\signcheck.errors.log + $(SignCheckInputDir) + @(ItemsToSign) - - + + + + - - - - - - - - - - - - - - + - diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/Sign.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/Sign.proj index 539570dfd47..6c09f050779 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/Sign.proj +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/Sign.proj @@ -4,52 +4,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - false - - - - Signing - - - - + true + + $(_VSInstallDir)\MSBuild\15.0\Bin\msbuild.exe - - - - diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/Sign.props b/src/Microsoft.DotNet.Arcade.Sdk/tools/Sign.props new file mode 100644 index 00000000000..64c419aae1b --- /dev/null +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/Sign.props @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + false + + Signing + + + + + diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/Tools.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/Tools.proj index 5837177c853..4835b23520a 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/Tools.proj +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/Tools.proj @@ -26,6 +26,7 @@ + diff --git a/src/Microsoft.DotNet.AzureDevOps.Build/Microsoft.DotNet.AzureDevOps.Build.csproj b/src/Microsoft.DotNet.AzureDevOps.Build/Microsoft.DotNet.AzureDevOps.Build.csproj new file mode 100644 index 00000000000..d43a0b00180 --- /dev/null +++ b/src/Microsoft.DotNet.AzureDevOps.Build/Microsoft.DotNet.AzureDevOps.Build.csproj @@ -0,0 +1,33 @@ + + + + + netcoreapp2.1;net472 + true + + This package provides support for Azure DevOps builds + true + MSBuildSdk + true + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Microsoft.DotNet.AzureDevOps.Build/src/GenerateBuildManifest.cs b/src/Microsoft.DotNet.AzureDevOps.Build/src/GenerateBuildManifest.cs new file mode 100644 index 00000000000..eeb83c925a5 --- /dev/null +++ b/src/Microsoft.DotNet.AzureDevOps.Build/src/GenerateBuildManifest.cs @@ -0,0 +1,62 @@ +using Microsoft.Build.Construction; +using Microsoft.Build.Framework; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.DotNet.AzureDevOps.Build +{ + public class GenerateAzureDevOpsBuildManifest : Microsoft.Build.Utilities.Task + { + [Required] + public string AzureDevOpsCollectionUri { get; set; } + [Required] + public string AzureDevOpsProject { get; set; } + [Required] + public int AzureDevOpsBuildId { get; set; } + [Required] + public string ManifestPath { get; set; } + public ITaskItem[] ItemsToSign { get; set; } + public ITaskItem[] StrongNameSignInfo { get; set; } + public ITaskItem[] FileSignInfo { get; set; } + public ITaskItem[] FileExtensionSignInfo { get; set; } + public override bool Execute() + { + ProjectRootElement project = ProjectRootElement.Create(); + var propertyGroup = project.CreatePropertyGroupElement(); + project.AppendChild(propertyGroup); + propertyGroup.AddProperty("AzureDevOpsCollectionUri", AzureDevOpsCollectionUri); + propertyGroup.AddProperty("AzureDevOpsProject", AzureDevOpsProject); + propertyGroup.AddProperty("AzureDevOpsBuildId", AzureDevOpsBuildId.ToString()); + var itemGroup = project.CreateItemGroupElement(); + project.AppendChild(itemGroup); + if (ItemsToSign != null) + { + foreach (var itemToSign in ItemsToSign) + { + var filename = itemToSign.ItemSpec.Replace('\\', '/'); + { + var metadata = itemToSign.CloneCustomMetadata() as Dictionary; + itemGroup.AddItem("ItemsToSign", Path.GetFileName(itemToSign.ItemSpec), metadata); + } + } + } + foreach (var signInfo in StrongNameSignInfo) + { + itemGroup.AddItem("StrongNameSignInfo", Path.GetFileName(signInfo.ItemSpec), signInfo.CloneCustomMetadata() as Dictionary); + } + foreach (var signInfo in FileSignInfo) + { + itemGroup.AddItem("FileSignInfo", signInfo.ItemSpec, signInfo.CloneCustomMetadata() as Dictionary); + } + foreach (var signInfo in FileExtensionSignInfo) + { + itemGroup.AddItem("FileExtensionSignInfo", signInfo.ItemSpec, signInfo.CloneCustomMetadata() as Dictionary); + } + project.Save(ManifestPath); + return true; + } + } +} diff --git a/src/SignCheck/SignCheck/Microsoft.DotNet.SignCheck.csproj b/src/SignCheck/SignCheck/Microsoft.DotNet.SignCheck.csproj index 2fe835e63e6..10fde204ab2 100644 --- a/src/SignCheck/SignCheck/Microsoft.DotNet.SignCheck.csproj +++ b/src/SignCheck/SignCheck/Microsoft.DotNet.SignCheck.csproj @@ -23,6 +23,7 @@ + diff --git a/src/SignCheck/SignCheck/SignCheck.cs b/src/SignCheck/SignCheck/SignCheck.cs index c2787e35881..d29d19e91b1 100644 --- a/src/SignCheck/SignCheck/SignCheck.cs +++ b/src/SignCheck/SignCheck/SignCheck.cs @@ -188,7 +188,10 @@ private List GetInputFilesFromOptions() { var inputFiles = new List(); var downloadFiles = new List(); - + if (Options.InputFiles == null) + { + return inputFiles; + } foreach (string inputFile in Options.InputFiles) { Uri uriResult; @@ -287,7 +290,6 @@ private List GetInputFilesFromOptions() { inputFiles.Remove(Path.GetFullPath(Options.LogFile)); } - return inputFiles; } @@ -369,7 +371,7 @@ public void GenerateExclusionsFile(StreamWriter writer, IEnumerable LogVerbosity.Normal ? DetailKeys.ResultKeysVerbose : DetailKeys.ResultKeysNormal; - if (InputFiles.Count() > 0) + if (InputFiles != null && InputFiles.Count() > 0) { DateTime startTime = DateTime.Now; IEnumerable results = signatureVerificationManager.VerifyFiles(InputFiles); diff --git a/src/SignCheck/SignCheck/SignCheckTask.cs b/src/SignCheck/SignCheck/SignCheckTask.cs new file mode 100644 index 00000000000..0cf6980e6ed --- /dev/null +++ b/src/SignCheck/SignCheck/SignCheckTask.cs @@ -0,0 +1,151 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.SignCheck.Logging; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace SignCheck +{ + public class SignCheckTask : Microsoft.Build.Utilities.Task + { + public bool EnableJarSignatureVerification + { + get; + set; + } + public bool EnableXmlSignatureVerification + { + get; + set; + } + public string FileStatus + { + get; + set; + } + + public string ExclusionsOutput + { + get; + set; + } + + public bool SkipTimestamp + { + get; + set; + } + public bool Recursive + { + get; + set; + } + + public bool VerifyStrongName + { + get; + set; + } + public string ExclusionsFile + { + get; + set; + } + public ITaskItem[] InputFiles + { + get; + set; + } + [Required] + public string LogFile + { + get; + set; + } + [Required] + public string ErrorLogFile + { + get; + set; + } + public string Verbosity + { + get; + set; + } + public string ArtifactFolder + { + get; + set; + } + + public override bool Execute() + { + Options options = new Options(); + options.EnableJarSignatureVerification = EnableJarSignatureVerification; + options.EnableXmlSignatureVerification = EnableXmlSignatureVerification; + options.ExclusionsFile = ExclusionsFile; + options.ExclusionsOutput = ExclusionsOutput; + string[] filestatuses = FileStatus.Split(';', ','); + options.FileStatus = filestatuses; + options.Recursive = Recursive; + options.TraverseSubFolders = Recursive; + options.SkipTimestamp = SkipTimestamp; + options.VerifyStrongName = VerifyStrongName; + options.LogFile = LogFile; + options.ErrorLogFile = ErrorLogFile; + + List inputFiles = new List(); + if (InputFiles != null) + { + ArtifactFolder = ArtifactFolder ?? Environment.CurrentDirectory; + SearchOption fileSearchOptions = Recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly; + + foreach (var checkFile in InputFiles.Select(s => s.ItemSpec).ToArray()) + { + if (Path.IsPathRooted(checkFile)) + { + inputFiles.Add(checkFile); + } + else + { + var matchedFiles = Directory.GetFiles(ArtifactFolder, checkFile, fileSearchOptions); + + if(matchedFiles.Length == 1) + { + inputFiles.Add(matchedFiles[0]); + } + else if(matchedFiles.Length == 0) + { + Log.LogError($"Unable to find file '{checkFile}' in folder '{ArtifactFolder}'. Try specifying 'Recursive=true` to include subfolders"); + } + else if (matchedFiles.Length > 1) + { + Log.LogError($"found multiple files matching pattern '{checkFile}'"); + foreach(var file in matchedFiles) + { + Log.LogError($" - {file}"); + } + } + } + } + } + options.InputFiles = inputFiles.ToArray(); + + if(Enum.TryParse(Verbosity, out LogVerbosity verbosity)) + { + options.Verbosity = verbosity; + } + + var sc = new SignCheck(new string[] { }); + sc.Options = options; + int result = sc.Run(); + return (result == 0 && !Log.HasLoggedErrors); + } + } +}