Skip to content

Improve prebuilt baselining: warn on error by default, save all project.assets.json #776

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dagood merged 4 commits into from
Sep 18, 2018
Merged

Improve prebuilt baselining: warn on error by default, save all project.assets.json #776

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0627f66
By default, continue on prebuilt baseline error
dagood Sep 16, 2018
62ede92
Save all potential reports for more diagnosability
dagood Sep 16, 2018
2c43a36
Create zip with all project.assets.json files
dagood Sep 17, 2018
52cb335
Set MSBUILDDEBUGPATH env var for all builds
dagood Sep 17, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions build.proj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
<MakeDir Directories="$(SourceBuiltPackagesPath)" />
<MakeDir Directories="$(SourceBuiltAssetsDir)" />
<MakeDir Directories="$(LocalBlobStorageRoot)" />
<MakeDir Directories="$(MSBuildDebugPathTargetDir)" />
</Target>

<Target Name="InitBuild">
Expand Down
2 changes: 2 additions & 0 deletions dir.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,12 +62,14 @@
<RestoreSourcePropsPath>$(IntermediatePath)RestoreSources.props</RestoreSourcePropsPath>
<PackageVersionPropsPath>$(IntermediatePath)PackageVersions.props</PackageVersionPropsPath>
<LoggingDir>$(BaseOutputPath)logs/</LoggingDir>
<MSBuildDebugPathTargetDir>$(BaseOutputPath)msbuild-debug/</MSBuildDebugPathTargetDir>
<!-- Dir where git info is generated during online builds. -->
<GitInfoOutputDir>$(BaseOutputPath)git-info/</GitInfoOutputDir>
<!-- Dir where git info is placed inside the tarball. -->
<GitInfoOfflineDir>$(ProjectDir)git-info/</GitInfoOfflineDir>
<PackageReportDir>$(BaseOutputPath)prebuilt-report/</PackageReportDir>
<PackageReportDataFile>$(PackageReportDir)prebuilt-usage.xml</PackageReportDataFile>
<ProjectAssetsJsonArchiveFile>$(PackageReportDir)all-project-assets-json-files.zip</ProjectAssetsJsonArchiveFile>
<ProdConManifestFile>$(PackageReportDir)prodcon-build.xml</ProdConManifestFile>
<PoisonedReportFile>$(PackageReportDir)poisoned.txt</PoisonedReportFile>
<ConflictingPackageReportDir>$(BaseOutputPath)conflict-report/</ConflictingPackageReportDir>
Expand Down
14 changes: 8 additions & 6 deletions netci.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@ def addArchival(def job) {
// Grab these logs from all of those locations.
[ "", "source-build/", "tarball-output/"].each { logRoot ->
archivalSettings.addFiles("${logRoot}bin/logs/*")
archivalSettings.addFiles("${logRoot}bin/prebuilt-report/*")
archivalSettings.addFiles("${logRoot}bin/prebuilt-report/**/*")
Copy link
Copy Markdown
Contributor

@crummel crummel Sep 17, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this still pick up loose files in prebuilt-report? The XML summaries will still be going in there, right?

Copy link
Copy Markdown
Member Author

@dagood dagood Sep 17, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It should--** is meant to match no/current directory as well as all recursive subdirectories.

Copy link
Copy Markdown
Member Author

@dagood dagood Sep 17, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Testing this with https://ci.dot.net/job/dotnet_source-build/job/release_2.1/job/GenPRTest/job/Ubuntu16.04_Release_prtest/1/

(Edit: It confirms.)

archivalSettings.addFiles("${logRoot}bin/conflict-report/**/*")
archivalSettings.addFiles("${logRoot}bin/msbuild-debug/**/*")
archivalSettings.addFiles("${logRoot}src/**/*.binlog")
archivalSettings.addFiles("${logRoot}src/**/*.log")
archivalSettings.addFiles("${logRoot}init-tools.log")
Expand Down Expand Up @@ -58,11 +60,11 @@ def addBuildStepsAndSetMachineAffinity(def job, String os, String configuration)
steps {
if (os == "Windows_NT") {
batchFile("git submodule update --init --recursive");
batchFile(".\\build.cmd /p:Configuration=${configuration} ${loggingOptions}")
batchFile(".\\build.cmd /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}")
}
else {
shell("git submodule update --init --recursive");
shell("./build.sh /p:Configuration=${configuration} ${loggingOptions}");
shell("./build.sh /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}");
smokeTestExcludes = "";
if (os == "Fedora24" || os == "OSX10.12") {
// Dev certs doesn't seem to work in these platforms. https://github.com/dotnet/source-build/issues/560
Expand Down Expand Up @@ -132,7 +134,7 @@ def addPushJob(String project, String branch, String os, String configuration)
shell("cd ./source-build;./build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} ${loggingOptions}");
shell("cd ./source-build;./build-source-tarball.sh ../tarball-output --skip-build");

shell("cd ./tarball-output;./build.sh /p:Configuration=${configuration} ${loggingOptions}")
shell("cd ./tarball-output;./build.sh /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}")
shell("cd ./tarball-output;./smoke-test.sh --minimal --configuration ${configuration}")
}
}
Expand Down Expand Up @@ -178,13 +180,13 @@ def addPushJob(String project, String branch, String os, String configuration)
steps{
shell("cd ./source-build;git submodule update --init --recursive");
// First build the product itself
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home -v \$(pwd)/source-build:/opt/code --rm -w /opt/code ${imageName} /opt/code/build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} /p:ContinueOnPrebuiltBaselineError=true ${loggingOptions}");
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home -v \$(pwd)/source-build:/opt/code --rm -w /opt/code ${imageName} /opt/code/build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} ${loggingOptions}");
// Have to make this directory before volume-sharing it unlike non-docker build - existing directory is really only a warning in build-source-tarball.sh
shell("mkdir tarball-output");
// now build the tarball
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home --network none -v \$(pwd)/source-build:/opt/code -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/code ${imageName} /opt/code/build-source-tarball.sh /opt/tarball --skip-build");
// now build from the tarball offline and without access to the regular non-tarball build
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home --network none -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/build.sh /p:Configuration=${configuration} ${loggingOptions}");
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home --network none -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/build.sh /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}");
// finally, run a smoke-test on the result
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/smoke-test.sh --minimal --configuration ${configuration}");
}
Expand Down
3 changes: 3 additions & 0 deletions repos/dir.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@
<EnvironmentVariables Include="PB_IsStable=$(UseStableVersions)" Condition="'$(UseStableVersions)' != ''" />
<EnvironmentVariables Include="PackageVersionStamp=$(VersionStamp)" Condition="'$(VersionStamp)' != ''" />

<!-- If MSBuild exits early, make sure debug output like 'MSBuild_*.failure.txt' ends up in a place we can see it. -->
<EnvironmentVariables Include="MSBUILDDEBUGPATH=$(MSBuildDebugPathTargetDir)" />

<!-- Turn off node reuse for source build because repos use conflicting versions
of compilers which cause assembly load errors.
See https://github.com/dotnet/source-build/issues/541 -->
Expand Down
14 changes: 11 additions & 3 deletions repos/dir.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,11 +192,13 @@
<PropertyGroup>
<_ReportDir>$(ConflictingPackageReportDir)before-$(RepositoryName)/</_ReportDir>
<_ReportDataFile>$(_ReportDir)usage.xml</_ReportDataFile>
<_ProjectAssetsJsonArchiveFile>$(_ReportDir)all-project-assets-json-files.zip</_ProjectAssetsJsonArchiveFile>
</PropertyGroup>

<WritePackageUsageData NuGetPackageInfos="@(ConflictingPackageInfos)"
RootDir="$(ProjectDir)"
DataFile="$(_ReportDataFile)" />
DataFile="$(_ReportDataFile)"
ProjectAssetsJsonArchiveFile="$(_ProjectAssetsJsonArchiveFile)" />

<WriteUsageReports DataFile="$(_ReportDataFile)"
OutputDirectory="$(_ReportDir)" />
Expand Down Expand Up @@ -385,7 +387,8 @@
TargetRid="$(TargetRid)"
ProjectDirectories="@(ProjectDirectories)"
RootDir="$(ProjectDir)"
DataFile="$(PackageReportDataFile)" />
DataFile="$(PackageReportDataFile)"
ProjectAssetsJsonArchiveFile="$(ProjectAssetsJsonArchiveFile)" />
</Target>

<Target Name="GetAllProjectDirectories">
Expand All @@ -407,7 +410,12 @@
<BaselineDataFile Condition="'$(OfflineBuild)' == 'true'">$(BaselineDataFile)offline.xml</BaselineDataFile>
<BaselineDataFile Condition="'$(OfflineBuild)' != 'true'">$(BaselineDataFile)online.xml</BaselineDataFile>

<ContinueOnPrebuiltBaselineError Condition="'$(ContinueOnPrebuiltBaselineError)' == ''">false</ContinueOnPrebuiltBaselineError>
<FailOnPrebuiltBaselineError Condition="'$(FailOnPrebuiltBaselineError)' == ''">false</FailOnPrebuiltBaselineError>
</PropertyGroup>

<PropertyGroup Condition="'$(ContinueOnPrebuiltBaselineError)' == ''">
<ContinueOnPrebuiltBaselineError>false</ContinueOnPrebuiltBaselineError>
<ContinueOnPrebuiltBaselineError Condition="'$(FailOnPrebuiltBaselineError)' != 'true'">true</ContinueOnPrebuiltBaselineError>
</PropertyGroup>

<ValidateUsageAgainstBaseline
Expand Down
36 changes: 36 additions & 0 deletions tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks/UsageReport/WritePackageUsageData.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Threading.Tasks;
using Task = Microsoft.Build.Utilities.Task;
Expand Down Expand Up @@ -68,6 +69,12 @@ public class WritePackageUsageData : Task
[Required]
public string DataFile { get; set; }

/// <summary>
/// If passed, the path of the archive file to generate that includes a copy of all
/// project.asset.json files found.
/// </summary>
public string ProjectAssetsJsonArchiveFile { get; set; }

public override bool Execute()
{
DateTime startTime = DateTime.Now;
Expand Down Expand Up @@ -124,6 +131,35 @@ public override bool Execute()
"project.assets.json",
SearchOption.AllDirectories);

if (!string.IsNullOrEmpty(ProjectAssetsJsonArchiveFile))
{
Log.LogMessage(MessageImportance.Low, "Archiving project.assets.json files...");

Directory.CreateDirectory(Path.GetDirectoryName(ProjectAssetsJsonArchiveFile));

using (var projectAssetArchive = new ZipArchive(
File.Open(
ProjectAssetsJsonArchiveFile,
FileMode.Create,
FileAccess.ReadWrite),
ZipArchiveMode.Create))
{
// Only one entry can be open at a time, so don't do this during the Parallel
// ForEach later.
foreach (var file in assetFiles)
{
string relativePath = file.Substring(RootDir.Length);
using (var stream = File.OpenRead(file))
using (Stream entryWriter = projectAssetArchive
.CreateEntry(relativePath, CompressionLevel.Optimal)
.Open())
{
stream.CopyTo(entryWriter);
}
}
}
}

Log.LogMessage(MessageImportance.Low, "Reading usage info...");

var usages = new ConcurrentBag<Usage>();
Expand Down