From 0627f668d2043ebf2d18c29e3373b4892d0ecff9 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Sun, 16 Sep 2018 17:07:33 -0500 Subject: [PATCH 1/4] By default, continue on prebuilt baseline error --- netci.groovy | 10 +++++----- repos/dir.targets | 7 ++++++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/netci.groovy b/netci.groovy index e7af288a82..e3f5eccf85 100644 --- a/netci.groovy +++ b/netci.groovy @@ -58,11 +58,11 @@ def addBuildStepsAndSetMachineAffinity(def job, String os, String configuration) steps { if (os == "Windows_NT") { batchFile("git submodule update --init --recursive"); - batchFile(".\\build.cmd /p:Configuration=${configuration} ${loggingOptions}") + batchFile(".\\build.cmd /p:FailOnPrebuiltBaselineError=true /p:Configuration=${configuration} ${loggingOptions}") } else { shell("git submodule update --init --recursive"); - shell("./build.sh /p:Configuration=${configuration} ${loggingOptions}"); + shell("./build.sh /p:FailOnPrebuiltBaselineError=true /p:Configuration=${configuration} ${loggingOptions}"); smokeTestExcludes = ""; if (os == "Fedora24" || os == "OSX10.12") { // Dev certs doesn't seem to work in these platforms. https://github.com/dotnet/source-build/issues/560 @@ -132,7 +132,7 @@ def addPushJob(String project, String branch, String os, String configuration) shell("cd ./source-build;./build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} ${loggingOptions}"); shell("cd ./source-build;./build-source-tarball.sh ../tarball-output --skip-build"); - shell("cd ./tarball-output;./build.sh /p:Configuration=${configuration} ${loggingOptions}") + shell("cd ./tarball-output;./build.sh /p:FailOnPrebuiltBaselineError=true /p:Configuration=${configuration} ${loggingOptions}") shell("cd ./tarball-output;./smoke-test.sh --minimal --configuration ${configuration}") } } @@ -178,13 +178,13 @@ def addPushJob(String project, String branch, String os, String configuration) steps{ shell("cd ./source-build;git submodule update --init --recursive"); // First build the product itself - shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home -v \$(pwd)/source-build:/opt/code --rm -w /opt/code ${imageName} /opt/code/build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} /p:ContinueOnPrebuiltBaselineError=true ${loggingOptions}"); + shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home -v \$(pwd)/source-build:/opt/code --rm -w /opt/code ${imageName} /opt/code/build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} ${loggingOptions}"); // Have to make this directory before volume-sharing it unlike non-docker build - existing directory is really only a warning in build-source-tarball.sh shell("mkdir tarball-output"); // now build the tarball shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home --network none -v \$(pwd)/source-build:/opt/code -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/code ${imageName} /opt/code/build-source-tarball.sh /opt/tarball --skip-build"); // now build from the tarball offline and without access to the regular non-tarball build - shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home --network none -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/build.sh /p:Configuration=${configuration} ${loggingOptions}"); + shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home --network none -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/build.sh /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}"); // finally, run a smoke-test on the result shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/smoke-test.sh --minimal --configuration ${configuration}"); } diff --git a/repos/dir.targets b/repos/dir.targets index 04cd9f7237..26dd8b59b6 100644 --- a/repos/dir.targets +++ b/repos/dir.targets @@ -407,7 +407,12 @@ $(BaselineDataFile)offline.xml $(BaselineDataFile)online.xml - false + false + + + + false + true Date: Sun, 16 Sep 2018 17:08:49 -0500 Subject: [PATCH 2/4] Save all potential reports for more diagnosability --- netci.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/netci.groovy b/netci.groovy index e3f5eccf85..cf7ac73441 100644 --- a/netci.groovy +++ b/netci.groovy @@ -13,7 +13,8 @@ def addArchival(def job) { // Grab these logs from all of those locations. [ "", "source-build/", "tarball-output/"].each { logRoot -> archivalSettings.addFiles("${logRoot}bin/logs/*") - archivalSettings.addFiles("${logRoot}bin/prebuilt-report/*") + archivalSettings.addFiles("${logRoot}bin/prebuilt-report/**/*") + archivalSettings.addFiles("${logRoot}bin/conflict-report/**/*") archivalSettings.addFiles("${logRoot}src/**/*.binlog") archivalSettings.addFiles("${logRoot}src/**/*.log") archivalSettings.addFiles("${logRoot}init-tools.log") From 2c43a3635e1d2c9952606a77f02b49e3a5997b9f Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Mon, 17 Sep 2018 09:28:47 -0500 Subject: [PATCH 3/4] Create zip with all project.assets.json files --- dir.props | 1 + netci.groovy | 6 ++-- repos/dir.targets | 7 ++-- .../UsageReport/WritePackageUsageData.cs | 36 +++++++++++++++++++ 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/dir.props b/dir.props index 9f6f8e636e..b9cbe2a171 100644 --- a/dir.props +++ b/dir.props @@ -68,6 +68,7 @@ $(ProjectDir)git-info/ $(BaseOutputPath)prebuilt-report/ $(PackageReportDir)prebuilt-usage.xml + $(PackageReportDir)all-project-assets-json-files.zip $(PackageReportDir)prodcon-build.xml $(PackageReportDir)poisoned.txt $(BaseOutputPath)conflict-report/ diff --git a/netci.groovy b/netci.groovy index cf7ac73441..0b9c732214 100644 --- a/netci.groovy +++ b/netci.groovy @@ -59,11 +59,11 @@ def addBuildStepsAndSetMachineAffinity(def job, String os, String configuration) steps { if (os == "Windows_NT") { batchFile("git submodule update --init --recursive"); - batchFile(".\\build.cmd /p:FailOnPrebuiltBaselineError=true /p:Configuration=${configuration} ${loggingOptions}") + batchFile(".\\build.cmd /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}") } else { shell("git submodule update --init --recursive"); - shell("./build.sh /p:FailOnPrebuiltBaselineError=true /p:Configuration=${configuration} ${loggingOptions}"); + shell("./build.sh /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}"); smokeTestExcludes = ""; if (os == "Fedora24" || os == "OSX10.12") { // Dev certs doesn't seem to work in these platforms. https://github.com/dotnet/source-build/issues/560 @@ -133,7 +133,7 @@ def addPushJob(String project, String branch, String os, String configuration) shell("cd ./source-build;./build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} ${loggingOptions}"); shell("cd ./source-build;./build-source-tarball.sh ../tarball-output --skip-build"); - shell("cd ./tarball-output;./build.sh /p:FailOnPrebuiltBaselineError=true /p:Configuration=${configuration} ${loggingOptions}") + shell("cd ./tarball-output;./build.sh /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}") shell("cd ./tarball-output;./smoke-test.sh --minimal --configuration ${configuration}") } } diff --git a/repos/dir.targets b/repos/dir.targets index 26dd8b59b6..d3386477d8 100644 --- a/repos/dir.targets +++ b/repos/dir.targets @@ -192,11 +192,13 @@ <_ReportDir>$(ConflictingPackageReportDir)before-$(RepositoryName)/ <_ReportDataFile>$(_ReportDir)usage.xml + <_ProjectAssetsJsonArchiveFile>$(_ReportDir)all-project-assets-json-files.zip + DataFile="$(_ReportDataFile)" + ProjectAssetsJsonArchiveFile="$(_ProjectAssetsJsonArchiveFile)" /> @@ -385,7 +387,8 @@ TargetRid="$(TargetRid)" ProjectDirectories="@(ProjectDirectories)" RootDir="$(ProjectDir)" - DataFile="$(PackageReportDataFile)" /> + DataFile="$(PackageReportDataFile)" + ProjectAssetsJsonArchiveFile="$(ProjectAssetsJsonArchiveFile)" /> diff --git a/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks/UsageReport/WritePackageUsageData.cs b/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks/UsageReport/WritePackageUsageData.cs index 60bc3a0001..72f82cf35b 100644 --- a/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks/UsageReport/WritePackageUsageData.cs +++ b/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks/UsageReport/WritePackageUsageData.cs @@ -11,6 +11,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; using System.Threading.Tasks; using Task = Microsoft.Build.Utilities.Task; @@ -68,6 +69,12 @@ public class WritePackageUsageData : Task [Required] public string DataFile { get; set; } + /// + /// If passed, the path of the archive file to generate that includes a copy of all + /// project.asset.json files found. + /// + public string ProjectAssetsJsonArchiveFile { get; set; } + public override bool Execute() { DateTime startTime = DateTime.Now; @@ -124,6 +131,35 @@ public override bool Execute() "project.assets.json", SearchOption.AllDirectories); + if (!string.IsNullOrEmpty(ProjectAssetsJsonArchiveFile)) + { + Log.LogMessage(MessageImportance.Low, "Archiving project.assets.json files..."); + + Directory.CreateDirectory(Path.GetDirectoryName(ProjectAssetsJsonArchiveFile)); + + using (var projectAssetArchive = new ZipArchive( + File.Open( + ProjectAssetsJsonArchiveFile, + FileMode.Create, + FileAccess.ReadWrite), + ZipArchiveMode.Create)) + { + // Only one entry can be open at a time, so don't do this during the Parallel + // ForEach later. + foreach (var file in assetFiles) + { + string relativePath = file.Substring(RootDir.Length); + using (var stream = File.OpenRead(file)) + using (Stream entryWriter = projectAssetArchive + .CreateEntry(relativePath, CompressionLevel.Optimal) + .Open()) + { + stream.CopyTo(entryWriter); + } + } + } + } + Log.LogMessage(MessageImportance.Low, "Reading usage info..."); var usages = new ConcurrentBag(); From 52cb3353f8bd3d26f32d751975cb92cefabf59f9 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Mon, 17 Sep 2018 18:01:23 -0500 Subject: [PATCH 4/4] Set MSBUILDDEBUGPATH env var for all builds Save contents in CI in case the outputs help troubleshooting. --- build.proj | 1 + dir.props | 1 + netci.groovy | 1 + repos/dir.props | 3 +++ 4 files changed, 6 insertions(+) diff --git a/build.proj b/build.proj index 2edac3d8e1..56f2c5599d 100644 --- a/build.proj +++ b/build.proj @@ -17,6 +17,7 @@ + diff --git a/dir.props b/dir.props index b9cbe2a171..d6d09c3b9e 100644 --- a/dir.props +++ b/dir.props @@ -62,6 +62,7 @@ $(IntermediatePath)RestoreSources.props $(IntermediatePath)PackageVersions.props $(BaseOutputPath)logs/ + $(BaseOutputPath)msbuild-debug/ $(BaseOutputPath)git-info/ diff --git a/netci.groovy b/netci.groovy index 0b9c732214..3e6736debb 100644 --- a/netci.groovy +++ b/netci.groovy @@ -15,6 +15,7 @@ def addArchival(def job) { archivalSettings.addFiles("${logRoot}bin/logs/*") archivalSettings.addFiles("${logRoot}bin/prebuilt-report/**/*") archivalSettings.addFiles("${logRoot}bin/conflict-report/**/*") + archivalSettings.addFiles("${logRoot}bin/msbuild-debug/**/*") archivalSettings.addFiles("${logRoot}src/**/*.binlog") archivalSettings.addFiles("${logRoot}src/**/*.log") archivalSettings.addFiles("${logRoot}init-tools.log") diff --git a/repos/dir.props b/repos/dir.props index bea8d507cd..e5b99b25d1 100644 --- a/repos/dir.props +++ b/repos/dir.props @@ -56,6 +56,9 @@ + + +