From 9c85652e952540ed8a008a1443edb9c89b931c04 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 14 Nov 2019 15:59:16 -0800 Subject: [PATCH 01/71] Convert libraries and installer to use a live CoreCLR. --- Directory.Build.targets | 5 +- eng/Version.Details.xml | 4 - eng/Versions.props | 1 - eng/liveBuilds.targets | 57 ++++++++ src/installer/Directory.Build.props | 2 +- .../pkg/projects/Directory.Build.targets | 4 +- .../src/localnetcoreapp.override.targets | 129 +++++++++--------- .../netcoreapp/src/netcoreapp.depproj | 56 +------- src/libraries/Directory.Build.props | 8 +- src/libraries/restore/runtime/runtime.depproj | 40 ++---- 10 files changed, 151 insertions(+), 155 deletions(-) create mode 100644 eng/liveBuilds.targets diff --git a/Directory.Build.targets b/Directory.Build.targets index 5180c31001b64c..1344122a8024d4 100644 --- a/Directory.Build.targets +++ b/Directory.Build.targets @@ -1,5 +1,4 @@ - + - - \ No newline at end of file + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index fcba33aff89a59..6df8d3b5d0d45c 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -28,10 +28,6 @@ https://github.com/dotnet/corefx 5cee7c97d602f294e27c582d4dab81ec388f1d7b - - https://github.com/dotnet/coreclr - 2c4fb3250989f014550882f5d165cdc36ebdbd08 - https://github.com/dotnet/coreclr 2c4fb3250989f014550882f5d165cdc36ebdbd08 diff --git a/eng/Versions.props b/eng/Versions.props index 38eeb0ba742136..46026ae78e7d51 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -50,7 +50,6 @@ 3.0.0 5.0.0-alpha1.19563.3 - 5.0.0-alpha1.19563.3 5.0.0-alpha1.19563.3 5.0.0-alpha.1.19563.6 diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets new file mode 100644 index 00000000000000..cb128489a28ce4 --- /dev/null +++ b/eng/liveBuilds.targets @@ -0,0 +1,57 @@ + + + + $([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant) + $(Configuration) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'coreclr', 'bin', 'Product', '$(OSGroup).$(TargetArchitecture).$(CoreCLRConfiguration)')) + x64 + x86 + x64 + + + + + + $([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)')) + + + $([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)','sharedFramework')) + $([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)','$(CoreCLRCrossTargetComponentDir)','sharedFramework')) + + + + + + + true + + + + + + + + runtime/$(RuntimeIdentifier)/native + + + + runtime/$(CoreCLRCrossTargetComponentDirName)_$(TargetArchitecture)/native + + + + + + + + + diff --git a/src/installer/Directory.Build.props b/src/installer/Directory.Build.props index 5032b57ce81ffa..1e1b456aae1560 100644 --- a/src/installer/Directory.Build.props +++ b/src/installer/Directory.Build.props @@ -402,7 +402,7 @@ lib - .so + .so .dll .dylib diff --git a/src/installer/pkg/projects/Directory.Build.targets b/src/installer/pkg/projects/Directory.Build.targets index b40cddb9dfd361..46319d4d1e34a6 100644 --- a/src/installer/pkg/projects/Directory.Build.targets +++ b/src/installer/pkg/projects/Directory.Build.targets @@ -336,4 +336,6 @@ - \ No newline at end of file + + + diff --git a/src/installer/pkg/projects/netcoreapp/src/localnetcoreapp.override.targets b/src/installer/pkg/projects/netcoreapp/src/localnetcoreapp.override.targets index 33d863469cb4ea..a4635909bd3e1e 100644 --- a/src/installer/pkg/projects/netcoreapp/src/localnetcoreapp.override.targets +++ b/src/installer/pkg/projects/netcoreapp/src/localnetcoreapp.override.targets @@ -1,57 +1,5 @@ - - - - - - $([MSBuild]::NormalizeDirectory('$(CoreCLROverridePath)')) - - - x64 - x86 - x64 - $([MSBuild]::NormalizeDirectory('$(CoreCLROverridePath)','sharedFramework')) - $([MSBuild]::NormalizeDirectory('$(CoreCLROverridePath)','$(CoreCLRCrossTargetComponentDir)','sharedFramework')) - - - - - - - true - - - - - - - - runtime/$(RuntimeIdentifier)/native - - - - runtime/$(CoreCLRCrossTargetComponentDirName)_$(TargetArchitecture)/native - - - - - - - - - - - + @@ -60,7 +8,7 @@ $(CoreFXOverridePath)/specs/$(MicrosoftPrivateCoreFxNETCoreAppPackage).nuspec $(CoreFXOverridePath)/specs/runtime.$(PackageRID).$(MicrosoftPrivateCoreFxNETCoreAppPackage).nuspec - + @@ -79,7 +27,7 @@ XmlInputPath="$(CoreFXSharedFrameworkImplementationPackageSpec)"> - + @@ -121,14 +69,71 @@ - - - <_runtimeDirectory>$(CoreCLROverridePath) - <_crossgenPath>$(CoreCLROverridePath)/crossgen$(ApplicationFileExtension) - <_CoreLibSharedDirectory>$(CoreCLROverridePath) - <_jitPath>$(CoreClrOverridePath)/$(_crossHostArch)/$(LibraryFilePrefix)clrjit$(LibraryFileExtension) + + + + <_runtimeCLR Include="$(_runtimePackageDir)**/$(LibraryFilePrefix)coreclr$(LibraryFileExtension)" /> + <_runtimeCoreLib Include="$(_runtimePackageDir)**/native/System.Private.CoreLib.dll" /> + <_fxSystemRuntime Include="$(_corefxPackageDir)**/System.Runtime.dll" /> + <_windowsWinMD Include="$(_winmdPackageDir)**/Windows.winmd" /> + <_diaSymReaderAssembly Include="$(_diaSymReaderPackageDir)**\Microsoft.DiaSymReader.Native.*.dll" /> + + + + <_runtimeDirectory>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', 'sharedFramework')) + <_crossGenPath>$([MSBuild]::NormalizePath('$(CoreCLRArtifactsPath)', '$(CoreCLRCrossTargetComponentDirName)', 'sharedFramework', 'crossgen$(ApplicationFileExtension)')) - + + <_coreLibDirectory>$(CoreCLRArtifactsPath) + + + + <_jitPath>$([MSBuild]::NormalizePath('$(CoreCLRArtifactsPath)', '$(CoreCLRCrossTargetComponentDirName)', 'sharedFramework', '$(LibraryFilePrefix)clrjit$(LibraryFileExtension)')) + + + <_fxLibDirectory>%(_fxSystemRuntime.RootDir)%(_fxSystemRuntime.Directory) + + + + <_windowsWinMDDirectory>%(_windowsWinMD.RootDir)%(_windowsWinMD.Directory) + + + + <_diaSymReaderToolDir>%(_diaSymReaderAssembly.RootDir)%(_diaSymReaderAssembly.Directory) + + + + + <_diaSymReaderToolDir>$(IntermediateOutputPath) + + + + + + + + + <_crossgenPlatformDirectories Include="%(_filesToCrossGen.RootDir)%(_filesToCrossGen.Directory)" /> + + <_crossgenPlatformDirectories Include="$(_runtimeDirectory)" /> + + <_crossgenPlatformDirectories Include="$(_coreLibDirectory)" /> + + + + + <_pathSeparatorEscaped>$([MSBuild]::Escape($([System.IO.Path]::PathSeparator.ToString()))) + <_crossgenPlatformAssemblies>@(_crossgenPlatformDirectories->'%(Identity)', '$(_pathSeparatorEscaped)') + + diff --git a/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj b/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj index 91ea562a339c7c..3b8806b39d8749 100644 --- a/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj +++ b/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj @@ -1,12 +1,13 @@ - - + + Microsoft.NETCore.App + true + false - @@ -31,53 +32,8 @@ Condition="'$(PackageTargetRuntime)' != ''" DependsOnTargets="GetCorePackagePaths" /> - - - <_coreFxVersionFile - Include="$(NuGetPackageRoot)$([System.String]::new('%(Reference.NuGetPackageId)').ToLowerInvariant())/%(Reference.NuGetPackageVersion)/version.txt" - Condition="$([System.String]::new('%(Reference.Identity)').ToLowerInvariant().Contains('system.runtime.dll'))" - Name="corefx" /> - <_coreClrVersionFile - Include="$(_runtimePackageDir)version.txt" - Name="coreclr" /> - - - - - - - - + - - - - <_ToolsToPackage Include="$(_runtimePackageDir)tools/**/*.*"/> - - $(_runtimePackageId) - - $([System.String]::new('tools/%(RecursiveDir)').TrimEnd('/')) - true - - - runtimes$(_crossDir)/native - true - - - - + diff --git a/src/libraries/Directory.Build.props b/src/libraries/Directory.Build.props index 33381d49d0b558..5d547ddd104892 100644 --- a/src/libraries/Directory.Build.props +++ b/src/libraries/Directory.Build.props @@ -90,7 +90,7 @@ - + true @@ -156,6 +156,7 @@ <_packageRID Condition="'$(PortableBuild)' == 'true'">$(_portableOS)-$(ArchGroup) $(_packageRID) $(RuntimeOS)-$(ArchGroup) + $(ArchGroup) @@ -169,6 +170,7 @@ true true + true - + false $(NoWarn);nullable - + diff --git a/src/libraries/restore/runtime/runtime.depproj b/src/libraries/restore/runtime/runtime.depproj index 1e01986e867155..bfb6ca5c83239b 100644 --- a/src/libraries/restore/runtime/runtime.depproj +++ b/src/libraries/restore/runtime/runtime.depproj @@ -5,6 +5,8 @@ $(ToolRuntimeRID) $(NoWarn);NU1603;NU1605 true + true + $(ConfigurationGroup) @@ -13,14 +15,11 @@ - 3.0.0-rc2.19462.14 4.6.0-rc2.19462.14 - - @@ -44,7 +43,7 @@ hostfxr libhostfxr - + true false @@ -75,39 +74,20 @@ - - - $(CoreCLROverridePath)/PDB - - - - - + + - + - + - <_CoreLibFile Include="@(ReferenceCopyLocalPaths)" Condition="'%(FileName)' == 'System.Private.CoreLib'" /> - - - <_CoreLibFilePath>%(_CoreLibFile.FullPath) - <_CoreLibPackagePath>$(_CoreLibFilePath.SubString(0, $(_CoreLibFilePath.IndexOf('runtimes')))) - - - - - - - - + @@ -122,5 +102,5 @@ - + From 88f21a920395f8e9b5b5bc48f229c2d6f2c14559 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 14 Nov 2019 18:08:32 -0800 Subject: [PATCH 02/71] Installer uses a live libraries build for DLLs. --- eng/liveBuilds.targets | 37 ++++++++-- .../src/localnetcoreapp.override.targets | 70 ------------------- .../netcoreapp/src/netcoreapp.depproj | 24 ++++++- src/libraries/Directory.Build.targets | 6 +- 4 files changed, 56 insertions(+), 81 deletions(-) diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets index cb128489a28ce4..864cdf5d321b9c 100644 --- a/eng/liveBuilds.targets +++ b/eng/liveBuilds.targets @@ -3,7 +3,13 @@ $([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant) $(Configuration) + $(Configuration) $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'coreclr', 'bin', 'Product', '$(OSGroup).$(TargetArchitecture).$(CoreCLRConfiguration)')) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'ref', 'microsoft.netcore.app', '$(LibrariesConfiguration)')) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'pkg', 'netcoreapp', 'runtime', '$(LibrariesConfiguration)')) + + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'native', 'netcoreapp-$(OSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) + x64 x86 x64 @@ -40,18 +46,35 @@ $(CoreCLRCrossTargetComponentDir)PDB/*.dbg; $(CoreCLRCrossTargetComponentDir)PDB/*.dwarf" /> - - runtime/$(RuntimeIdentifier)/native - - runtime/$(CoreCLRCrossTargetComponentDirName)_$(TargetArchitecture)/native + - - + + + + + + + + + + + - + + diff --git a/src/installer/pkg/projects/netcoreapp/src/localnetcoreapp.override.targets b/src/installer/pkg/projects/netcoreapp/src/localnetcoreapp.override.targets index a4635909bd3e1e..0e6ae6b8c293fc 100644 --- a/src/installer/pkg/projects/netcoreapp/src/localnetcoreapp.override.targets +++ b/src/installer/pkg/projects/netcoreapp/src/localnetcoreapp.override.targets @@ -1,74 +1,4 @@ - - - - - - - $(CoreFXOverridePath)/specs/$(MicrosoftPrivateCoreFxNETCoreAppPackage).nuspec - $(CoreFXOverridePath)/specs/runtime.$(PackageRID).$(MicrosoftPrivateCoreFxNETCoreAppPackage).nuspec - - - - - - - - - - - - - - - - - - - - - @(CoreFXReferenceItems) - - - - - - - @(CoreFXReferenceCopyLocalItems) - - - - true - - - - - - - - <_coreFXOverrideDocFiles - Condition="'$(CoreFXOverridePath)' != '' And '%(NuGetPackageId)' == '$(MicrosoftPrivateCoreFxNETCoreAppPackage)'" - Include="@(OverriddenFrameworkReferenceFiles->'$(CoreFXOverridePath)/../../bin/docs/%(FileName).xml')" /> - <_docFilesToPackage Include="@(_coreFXOverrideDocFiles)" Condition="Exists('%(Identity)')" /> - - - diff --git a/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj b/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj index 3b8806b39d8749..8e8506513bf4ea 100644 --- a/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj +++ b/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj @@ -7,7 +7,6 @@ - @@ -34,6 +33,29 @@ + + + + runtime/$(PackageRID)/native + + + + + + + + + + + + + + + + + + + diff --git a/src/libraries/Directory.Build.targets b/src/libraries/Directory.Build.targets index 26eb539d668a35..7a51a358386d9b 100644 --- a/src/libraries/Directory.Build.targets +++ b/src/libraries/Directory.Build.targets @@ -7,7 +7,7 @@ - + @@ -74,8 +74,8 @@ $(NETCoreAppPackageRefPath) $(NETCoreAppPackageRuntimePath) - $(NETCoreAppPackageRuntimePath)\..\runtime - $(RefRootPath)microsoft.netcore.app + $(ArtifactsBinDir)pkg\netcoreapp\runtime\$(ConfigurationGroup) + $(RefRootPath)microsoft.netcore.app\$(ConfigurationGroup) ILLinkTrimAssembly=true From 6164fa8631ec9f8dfb514074b3dea6d4c7d9a527 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 14 Nov 2019 18:17:34 -0800 Subject: [PATCH 03/71] Remove newline in path. --- eng/liveBuilds.targets | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets index 864cdf5d321b9c..4b0a6053c8c9ed 100644 --- a/eng/liveBuilds.targets +++ b/eng/liveBuilds.targets @@ -8,8 +8,7 @@ $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'ref', 'microsoft.netcore.app', '$(LibrariesConfiguration)')) $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'pkg', 'netcoreapp', 'runtime', '$(LibrariesConfiguration)')) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'native', 'netcoreapp-$(OSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) - + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'native', 'netcoreapp-$(OSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) x64 x86 x64 From 94a436be59e7bbf4bf2820f46b20516e2aa23bce Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Sun, 17 Nov 2019 12:01:25 -0800 Subject: [PATCH 04/71] Enable live-live builds to build required live dependencies when the dependencies are in the building subset/subset category. --- Directory.Build.props | 10 +++++++ eng/Subsets.props | 14 +++++++--- eng/liveBuilds.targets | 43 +++++++++++++++++++++++++---- src/coreclr/coreclr.proj | 4 +-- src/installer/Directory.Build.props | 6 ---- 5 files changed, 60 insertions(+), 17 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index c9024609663cbd..ade0ea1a8d1199 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -10,6 +10,10 @@ --> + + true + + @@ -34,6 +38,12 @@ $([MSBuild]::NormalizeDirectory('$(LibrariesProjectRoot)', 'System.Private.CoreLib', 'src')) + + + + + true + + - + libraries-installer-coreclr @@ -71,7 +75,7 @@ - + @@ -93,12 +97,14 @@ - + + - + + diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets index 4b0a6053c8c9ed..67a8a53a99d84b 100644 --- a/eng/liveBuilds.targets +++ b/eng/liveBuilds.targets @@ -4,18 +4,34 @@ $([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant) $(Configuration) $(Configuration) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'coreclr', 'bin', 'Product', '$(OSGroup).$(TargetArchitecture).$(CoreCLRConfiguration)')) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'ref', 'microsoft.netcore.app', '$(LibrariesConfiguration)')) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'pkg', 'netcoreapp', 'runtime', '$(LibrariesConfiguration)')) - - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'native', 'netcoreapp-$(OSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'coreclr', 'bin', 'Product', '$(OSGroup).$(TargetArchitecture).$(CoreCLRConfiguration)')) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'ref', 'microsoft.netcore.app', '$(LibrariesConfiguration)')) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'pkg', 'netcoreapp', 'runtime', '$(LibrariesConfiguration)')) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'native', 'netcoreapp-$(OSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) + $(CoreCLRArtifactsPathInTree) + $(LibrariesRefArtifactsPathInTree) + $(LibrariesBinArtifactsPathInTree) + $(LibrariesNativeArtifactsPathInTree) x64 x86 x64 + + true + + + + + $([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)')) @@ -54,6 +70,23 @@ + + true + true + true + + + + + diff --git a/src/coreclr/coreclr.proj b/src/coreclr/coreclr.proj index efdb3a58e65e8e..2b38d0368b3e5e 100644 --- a/src/coreclr/coreclr.proj +++ b/src/coreclr/coreclr.proj @@ -5,10 +5,10 @@ <_CoreClrBuildArg Condition="$([MSBuild]::IsOsPlatform(Windows))" Include="-skiptests" /> - <_CoreClrBuildArg Include="-$(Platform)" /> + <_CoreClrBuildArg Include="-$(TargetArchitecture)" /> <_CoreClrBuildArg Include="-$(Configuration.ToLower())" /> <_CoreClrBuildArg Condition="'$(ContinuousIntegrationBuild)' == 'true'" Include="-ci" /> - <_CoreClrBuildArg Condition="$([MSBuild]::IsOsPlatform(Windows)) and ('$(Platform)' == 'x86' or '$(Platform)' == 'x64') and '$(Configuration)' == 'Release'" Include="-enforcepgo" /> + <_CoreClrBuildArg Condition="$([MSBuild]::IsOsPlatform(Windows)) and ('$(TargetArchitecture)' == 'x86' or '$(TargetArchitecture)' == 'x64') and '$(Configuration)' == 'Release'" Include="-enforcepgo" /> <_CoreClrBuildArg Condition="'$(Configuration)' == 'Release'" Include="-stripsymbols" /> <_CoreClrBuildArg Condition="'$(OfficialBuildId)' == ''" Include="-officialbuildid=$(OfficialBuildId)" /> diff --git a/src/installer/Directory.Build.props b/src/installer/Directory.Build.props index 1e1b456aae1560..4cef6c389875e0 100644 --- a/src/installer/Directory.Build.props +++ b/src/installer/Directory.Build.props @@ -16,12 +16,6 @@ --> - - - ILLinkTrimAssembly=true From 39ab7ffa5a3f1b38d4680e8072da7a5a5b05cc78 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 18 Nov 2019 12:04:25 -0800 Subject: [PATCH 09/71] Fix typo in coreclr/tests/src/Common/Directory.Build.targets. --- .../tests/src/Common/Directory.Build.targets | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/coreclr/tests/src/Common/Directory.Build.targets b/src/coreclr/tests/src/Common/Directory.Build.targets index b613ab63368d3d..e589c1e48323af 100644 --- a/src/coreclr/tests/src/Common/Directory.Build.targets +++ b/src/coreclr/tests/src/Common/Directory.Build.targets @@ -11,20 +11,20 @@ DependsOnTargets="ResolveAssemblyReferences"> - - - - + + + + %(Identity) - - - + + + Date: Mon, 18 Nov 2019 12:07:33 -0800 Subject: [PATCH 10/71] Restore CoreFX shared framework from live build for CoreCLR test dependencies. --- .../test_dependencies/test_dependencies.csproj | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/coreclr/tests/src/Common/test_dependencies/test_dependencies.csproj b/src/coreclr/tests/src/Common/test_dependencies/test_dependencies.csproj index af83abd53df7b5..0ea0e7a27fc9e1 100644 --- a/src/coreclr/tests/src/Common/test_dependencies/test_dependencies.csproj +++ b/src/coreclr/tests/src/Common/test_dependencies/test_dependencies.csproj @@ -7,10 +7,12 @@ true true win-arm;win-arm64;win-x64;win-x86;$(TargetRid) + $(BuildArch) + $(BuildOS) + false + false - - @@ -25,4 +27,11 @@ $(SourceDir)Common\test_dependencies\obj\project.assets.json + + + + + + + From c2b930c98c8c647b376c4a5fecd0956d3cc2129f Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 18 Nov 2019 13:36:32 -0800 Subject: [PATCH 11/71] Remove dependency on Microsoft.NETCore.CoreCLR.TestDependencies package. --- .../tests/src/Common/test_runtime/test_runtime.csproj | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/coreclr/tests/src/Common/test_runtime/test_runtime.csproj b/src/coreclr/tests/src/Common/test_runtime/test_runtime.csproj index dbf6db252a49d7..6b35e3498a32ac 100644 --- a/src/coreclr/tests/src/Common/test_runtime/test_runtime.csproj +++ b/src/coreclr/tests/src/Common/test_runtime/test_runtime.csproj @@ -12,12 +12,17 @@ - + + + + + + From f73ad73b1872a8739db84f8a36f07aaf6f8f1738 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 18 Nov 2019 14:00:15 -0800 Subject: [PATCH 12/71] Enable referencing live builds of the OOB libraries by temporarily adding a new BinplaceConfiguration. Update test_dependencies in the CoreCLR tree to use that to restore OOB libraries. --- eng/liveBuilds.targets | 20 +++++++++++++------ .../test_dependencies.csproj | 7 +------ src/libraries/Directory.Build.targets | 7 ++++++- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets index f32369fb0aa241..e6db37b34638e2 100644 --- a/eng/liveBuilds.targets +++ b/eng/liveBuilds.targets @@ -6,11 +6,13 @@ $(Configuration) $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'coreclr', '$(OSGroup).$(TargetArchitecture).$(CoreCLRConfiguration)')) $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'ref', 'microsoft.netcore.app', '$(LibrariesConfiguration)')) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'pkg', 'netcoreapp', 'runtime', '$(OSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'pkg', 'netcoreapp', 'runtime', '$(OSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'pkg', 'oob', 'runtime', '$(OSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'native', 'netcoreapp-$(OSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) $(CoreCLRArtifactsPathInTree) $(LibrariesRefArtifactsPathInTree) - $(LibrariesBinArtifactsPathInTree) + $(LibrariesSharedFrameworkBinArtifactsPathInTree) + $(LibrariesOOBBinArtifactsPathInTree) $(LibrariesNativeArtifactsPathInTree) x64 x86 @@ -72,7 +74,8 @@ true - true + true + true true @@ -88,13 +91,18 @@ - + + + - - - - - diff --git a/src/libraries/Directory.Build.targets b/src/libraries/Directory.Build.targets index f83f6e97f94be4..4795ac3ec2c438 100644 --- a/src/libraries/Directory.Build.targets +++ b/src/libraries/Directory.Build.targets @@ -70,7 +70,7 @@ $(BuildConfigurationRefPath) $(RuntimePath) - + $(NETCoreAppPackageRefPath) $(NETCoreAppPackageRuntimePath) @@ -80,6 +80,11 @@ ILLinkTrimAssembly=true + + + $(ArtifactsBinDir)pkg\oob\runtime\$(_bc_OSGroup)-$(ConfigurationGroup)-$(ArchGroup) + + $(NETCoreAppTestSharedFrameworkPath) From eb0e52211cc46eb574f7fd48b934c8ef3f045052 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 18 Nov 2019 14:23:32 -0800 Subject: [PATCH 13/71] Remove reference to System.Drawing.Common from NETClientPrimitives (the reference is only needed at runtime and Core_Root already has System.Drawing.Common). --- .../COM/NETClients/Primitives/NETClientPrimitives.csproj | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/coreclr/tests/src/Interop/COM/NETClients/Primitives/NETClientPrimitives.csproj b/src/coreclr/tests/src/Interop/COM/NETClients/Primitives/NETClientPrimitives.csproj index c8ac76c7ac1245..0c7d448ed4fe84 100644 --- a/src/coreclr/tests/src/Interop/COM/NETClients/Primitives/NETClientPrimitives.csproj +++ b/src/coreclr/tests/src/Interop/COM/NETClients/Primitives/NETClientPrimitives.csproj @@ -24,8 +24,5 @@ - - $(MicrosoftPrivateCoreFxNETCoreAppVersion) - From 68e746bc08b713de4bbfba62e91283f69d0d4535 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 18 Nov 2019 14:50:21 -0800 Subject: [PATCH 14/71] Remove package-based checked CoreCLR + CoreFX test runs. In the live-live world, we'll run this test configuration by pointing the CoreFX test host build step at the artifacts from a checked CoreCLR build (possible in both local and CI builds). --- eng/Versions.props | 2 - eng/helixcorefxtests.proj | 291 ------------------ src/coreclr/build-test.cmd | 33 +- src/coreclr/build-test.sh | 46 +-- src/coreclr/tests/build.proj | 1 - src/coreclr/tests/scripts/patch-corefx.py | 263 ---------------- .../tests/src/Common/CoreFX/CoreFX.csproj | 150 --------- src/coreclr/tests/src/dirs.proj | 5 +- src/coreclr/tests/src/runtest.proj | 14 - src/libraries/restore/runtime/runtime.depproj | 7 +- 10 files changed, 5 insertions(+), 807 deletions(-) delete mode 100644 eng/helixcorefxtests.proj delete mode 100644 src/coreclr/tests/scripts/patch-corefx.py delete mode 100644 src/coreclr/tests/src/Common/CoreFX/CoreFX.csproj diff --git a/eng/Versions.props b/eng/Versions.props index 90d24b7faa17f8..e9e666d65c6648 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -53,7 +53,6 @@ 5.0.0-alpha1.19563.3 5.0.0-alpha.1.19563.6 - 5.0.0-alpha.1.19562.8 5.0.0-alpha.1.19563.6 5.0.0-alpha.1.19563.6 5.0.0-alpha.1.19563.6 @@ -106,7 +105,6 @@ optimization.linux-x64.ibc.corefx microsoft.private.intellisense Microsoft.DotNet.XUnitConsoleRunner - Microsoft.Private.CoreFx.NETCoreApp Microsoft.NETCore.Platforms Microsoft.NETCore.App optimization.IBC.CoreCLR diff --git a/eng/helixcorefxtests.proj b/eng/helixcorefxtests.proj deleted file mode 100644 index 7b14fb1d0ca70a..00000000000000 --- a/eng/helixcorefxtests.proj +++ /dev/null @@ -1,291 +0,0 @@ - - - - - - - - - - - - - - - - - - <_PropertiesToPass> - __BuildArch=$(__BuildArch); - __BuildOS=$(__BuildOS); - __BuildType=$(__BuildType); - Creator=$(_Creator); - HelixAccessToken=$(_HelixAccessToken); - HelixBuild=$(_HelixBuild); - HelixSource=$(_HelixSource); - HelixTargetQueues=$(_HelixTargetQueues); - HelixType=$(_HelixType); - PublishTestResults=$(_PublishTestResults); - RunCrossGen=$(_RunCrossGen); - TimeoutPerTestCollectionInMinutes=$(_TimeoutPerTestCollectionInMinutes) - - - - - - - - - <_Scenarios Include="$(_Scenarios.Split(','))" /> - - - <_ProjectsToBuild Include="$(MSBuildProjectFile)"> - $(_PropertiesToPass);Scenario=%(_Scenarios.Identity) - - - - - <_BuildInParallel>false - <_BuildInParallel Condition=" '@(_ProjectsToBuild->Count())' > '1' ">true - - - - - - - - - - $(_Creator) - $(_HelixAccessToken) - $(_HelixBuild) - $(_HelixSource) - $(_HelixTargetQueues) - $(_HelixType) - - - $(MSBuildThisFileDirectory)..\ - $(ProjectDir)artifacts\bin\coreclr - $(ProjectDir)artifacts\tests\coreclr\$(__BuildOS).$(__BuildArch).$(__BuildType)\ - true - - - $(ProjectDir)src\coreclr\tests\CoreFX\CoreFX.issues.rsp - - $(TestWorkingDir)testhost\ - $(TestWorkingDir)helix\ - - - shared\$(MicrosoftNETCoreAppPackage)\$(ProductVersion) - - https://dotnetfeed.blob.core.windows.net/dotnet-core - - - - - call RunTests.cmd -r %HELIX_CORRELATION_PAYLOAD% --rsp-file %HELIX_CORRELATION_PAYLOAD%\CoreFX.issues.rsp - ./RunTests.sh -r $HELIX_CORRELATION_PAYLOAD --rsp-file $HELIX_CORRELATION_PAYLOAD/CoreFX.issues.rsp - - - - - - - - - - - - SetStressModes_$(Scenario).cmd - SetStressModes_$(Scenario).sh - - - - - - - $(TestHostRootPath)$(TestEnvFileName) - - - - <_ProjectsToBuild Include="$(ProjectDir)src\coreclr\tests\testenvironment.proj"> - Scenario=$(Scenario);TestEnvFileName=$(TestEnvFilePath);TargetsWindows=$(TargetsWindows) - - - - - - - - - - - - - - <_Scenario Include="$(Scenarios.Split(','))" /> - <_ProjectsToBuild Include="$(MSBuildProjectFile)"> - Scenario=%(_Scenario.Identity) - - - - - - - - - - - - - - <_TargetGroup>netcoreapp - <_AssetManifestPath>$(TestAssetBlobFeedUrl)/corefx-tests/$(MicrosoftPrivateCoreFxNETCoreAppVersion)/$(__BuildOS).$(__BuildArch)/$(_TargetGroup)/corefx-test-assets.xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @(HelixPreCommand) - - - - $(__BuildArch) - - $(BuildType) - $(BuildType)-$(Scenario) - - CoreFX - $(TestRunNamePrefix)$(BuildOS) $(BuildArch) $(BuildType) @ - $(TestRunNamePrefix)$(BuildOS) $(BuildArch) $(BuildType) $(Scenario) @ - - - $(_PublishTestResults) - false - - true - true - true - true - - - - - - - - - - - - - - $(TestAssetBlobFeedUrl)/%(Identity) - $(HelixCommand) - $([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes))) - - - - - - - diff --git a/src/coreclr/build-test.cmd b/src/coreclr/build-test.cmd index f126181d88b681..532487b9792eda 100644 --- a/src/coreclr/build-test.cmd +++ b/src/coreclr/build-test.cmd @@ -237,7 +237,7 @@ if not exist "%VSINSTALLDIR%DIA SDK" goto NoDIA set __ExtraCmakeArgs="-DCMAKE_SYSTEM_VERSION=10.0" call "%__SourceDir%\pal\tools\gen-buildsys.cmd" "%__ProjectFilesDir%" "%__NativeTestIntermediatesDir%" %__VSVersion% %__BuildArch% !__ExtraCmakeArgs! - + if not !errorlevel! == 0 ( echo %__ErrMsgPrefix%%__MsgPrefix%Error: failed to generate native component build project! exit /b 1 @@ -474,37 +474,6 @@ if errorlevel 1 ( xcopy /s /y /i "%CORE_ROOT_STAGE%" "%CORE_ROOT%" -REM ========================================================================================= -REM === -REM === Create the test host necessary for running CoreFX tests. -REM === The test host includes a dotnet executable, system libraries and CoreCLR assemblies found in CORE_ROOT. -REM === -REM ========================================================================================= - -echo %__MsgPrefix%Building CoreFX test host - -set __BuildLogRootName=Tests_CoreFX_Testhost -set __BuildLog=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__BuildType%.log -set __BuildWrn=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__BuildType%.wrn -set __BuildErr=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__BuildType%.err -set __MsbuildLog=/flp:Verbosity=normal;LogFile="%__BuildLog%" -set __MsbuildWrn=/flp1:WarningsOnly;LogFile="%__BuildWrn%" -set __MsbuildErr=/flp2:ErrorsOnly;LogFile="%__BuildErr%" -set __Logging=!__MsbuildLog! !__MsbuildWrn! !__MsbuildErr! - -powershell -NoProfile -ExecutionPolicy ByPass -NoLogo -File "%__RepoRootDir%\eng\common\msbuild.ps1" %__ArcadeScriptArgs%^ - %__ProjectDir%\tests\src\runtest.proj /t:CreateTestHost /nodeReuse:false^ - /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true^ - /p:UsePartialNGENOptimization=false /maxcpucount^ - !__Logging! %__CommonMSBuildArgs% %RuntimeIdArg% %__PriorityArg% %__UnprocessedBuildArgs% -if errorlevel 1 ( - echo %__ErrMsgPrefix%%__MsgPrefix%Error: Create Test Host failed. Refer to the build log files for details: - echo %__BuildLog% - echo %__BuildWrn% - echo %__BuildErr% - exit /b 1 -) - REM ========================================================================================= REM === REM === Create test wrappers. diff --git a/src/coreclr/build-test.sh b/src/coreclr/build-test.sh index 3f0fc0634b5150..21af3fd3433960 100755 --- a/src/coreclr/build-test.sh +++ b/src/coreclr/build-test.sh @@ -132,16 +132,6 @@ generate_layout() fi } -patch_corefx_libraries() -{ - echo "${__MsgPrefix}Patching CORE_ROOT: '${CORE_ROOT}' with CoreFX libaries from enlistment '${__LocalCoreFXPath}" - - patchCoreFXArguments=("-clr_core_root" "${CORE_ROOT}" "-fx_root" "${__LocalCoreFXPath}" "-arch" "${__BuildArch}" "-build_type" "${__BuildType}") - scriptPath="$__ProjectDir/tests/scripts" - echo "python ${scriptPath}/patch-corefx.py ${patchCoreFXArguments[@]}" - $__Python "${scriptPath}/patch-corefx.py" "${patchCoreFXArguments[@]}" -} - precompile_coreroot_fx() { local overlayDir=$CORE_ROOT @@ -231,22 +221,6 @@ function is_skip_crossgen_test { return 1 } -generate_testhost() -{ - echo "${__MsgPrefix}Generating test host..." - - export TEST_HOST=$xUnitTestBinBase/testhost - - if [ -d "${TEST_HOST}" ]; then - rm -rf $TEST_HOST - fi - - mkdir -p $TEST_HOST - - build_MSBuild_projects "Tests_Generate_TestHost" "${__ProjectDir}/tests/src/runtest.proj" "Creating test host" "/t:CreateTestHost" -} - - build_Tests() { echo "${__MsgPrefix}Building Tests..." @@ -372,10 +346,6 @@ build_Tests() if [ $__SkipGenerateLayout != 1 ]; then generate_layout fi - - if [ ! -z "$__LocalCoreFXPath" ]; then - patch_corefx_libraries - fi } build_MSBuild_projects() @@ -574,10 +544,9 @@ usage_list=("-buildtestwrappersonly - only build the test wrappers.") usage_list+=("-copynativeonly: Only copy the native test binaries to the managed output. Do not build the native or managed tests.") usage_list+=("-crossgen - Precompiles the framework managed assemblies in coreroot.") usage_list+=("-generatelayoutonly - only pull down dependencies and build coreroot.") -usage_list+=("-generatetesthostonly - only pull down dependencies and build coreroot and the CoreFX testhost.") usage_list+=("-priority1 - include priority=1 tests in the build.") usage_list+=("-runtests - run tests after building them.") -usage_list+=("-skipgeneratelayout: Do not generate the Core_Root layout or the CoreFX testhost.") +usage_list+=("-skipgeneratelayout: Do not generate the Core_Root layout.") usage_list+=("-skiprestorepackages - skip package restore.") # Obtain the location of the bash script to figure out where the root of the repo is. @@ -630,16 +599,6 @@ handle_arguments() { __SkipGenerateLayout=1 ;; - localcorefxpath) - if [ -n "$2" ]; then - __LocalCoreFXPath="$2" - shift - else - echo "ERROR: 'localcorefxpath' requires a non-empty option argument" - exit 1 - fi - ;; - *) __UnprocessedBuildArgs+=("$1") ;; @@ -750,9 +709,6 @@ elif [ ! -z "$__BuildTestWrappersOnly" ]; then build_test_wrappers else generate_layout - if [ ! -z "$__GenerateTestHostOnly" ]; then - generate_testhost - fi fi if [ $? -ne 0 ]; then diff --git a/src/coreclr/tests/build.proj b/src/coreclr/tests/build.proj index ee9112a60804f5..94194931c1d05a 100644 --- a/src/coreclr/tests/build.proj +++ b/src/coreclr/tests/build.proj @@ -24,7 +24,6 @@ - diff --git a/src/coreclr/tests/scripts/patch-corefx.py b/src/coreclr/tests/scripts/patch-corefx.py deleted file mode 100644 index bbc44c1af07737..00000000000000 --- a/src/coreclr/tests/scripts/patch-corefx.py +++ /dev/null @@ -1,263 +0,0 @@ -#!/usr/bin/env python3 -# -# Licensed to the .NET Foundation under one or more agreements. -# The .NET Foundation licenses this file to you under the MIT license. -# See the LICENSE file in the project root for more information. -# -########################################################################## -########################################################################## -# -# Module: patch-corefx.py -# -# Notes: -# -# Script to overwrite the nuget downloaded corefx libraries with ones -# built from a local enlistment. -# -########################################################################## -########################################################################## - -import argparse -import distutils.dir_util -import os -import re -import shutil -import subprocess -import sys - -########################################################################## -# Globals -########################################################################## - -testing = False - -# This should be factored out of build.sh -Unix_name_map = { - 'Linux': 'Linux', - 'Darwin': 'OSX', - 'FreeBSD': 'FreeBSD', - 'OpenBSD': 'OpenBSD', - 'NetBSD': 'NetBSD', - 'SunOS': 'SunOS' -} - -Is_windows = (os.name == 'nt') - -########################################################################## -# Delete protocol -########################################################################## - -def del_rw(action, name, exc): - os.chmod(name, 0o651) - os.remove(name) - -########################################################################## -# Argument Parser -########################################################################## - -description = 'Tool to patch CoreFx tests on the CoreCLR repo' - -parser = argparse.ArgumentParser(description=description) - -parser.add_argument('-arch', dest='arch', default='x64') -parser.add_argument('-build_type', dest='build_type', default='Debug') -parser.add_argument('-clr_core_root', dest='clr_core_root', default=None) -parser.add_argument('-fx_root', dest='fx_root', default=None) - - -########################################################################## -# Helper Functions -########################################################################## - -def validate_args(args): - """ Validate all of the arguments parsed. - Args: - args (argparser.ArgumentParser): Args parsed by the argument parser. - Returns: - (arch, build_type, clr_core_root, fx_root,) - (str, str, str, str) - Notes: - If the arguments are valid then return them all in a tuple. If not, raise - an exception stating x argument is incorrect. - """ - - arch = args.arch - build_type = args.build_type - clr_core_root = args.clr_core_root - fx_root = args.fx_root - - def validate_arg(arg, check): - """ Validate an individual arg - Args: - arg (str|bool): argument to be validated - check (lambda: x-> bool): test that returns either True or False - : based on whether the check passes. - - Returns: - is_valid (bool): Is the argument valid? - """ - - helper = lambda item: item is not None and check(item) - - if not helper(arg): - raise Exception('Argument: %s is not valid.' % (arg)) - - valid_archs = ['x86', 'x64', 'arm', 'arm64'] - valid_build_types = ['Debug', 'Checked', 'Release'] - - arch = next((a for a in valid_archs if a.lower() == arch.lower()), arch) - build_type = next((b for b in valid_build_types if b.lower() == build_type.lower()), build_type) - - validate_arg(arch, lambda item: item in valid_archs) - validate_arg(build_type, lambda item: item in valid_build_types) - - if clr_core_root is None: - raise Exception('No clr_core_root argument provided') - else: - clr_core_root = os.path.normpath(clr_core_root) - validate_arg(clr_core_root, lambda item: os.path.isdir(clr_core_root)) - - if fx_root is None: - raise Exception('No fx_root argument provided') - else: - fx_root = os.path.normpath(fx_root) - - args = (arch, build_type, clr_core_root, fx_root) - - log('Configuration:') - log(' arch: %s' % arch) - log(' build_type: %s' % build_type) - log(' clr_core_root: %s' % clr_core_root) - log(' fx_root: %s' % fx_root) - - return args - -def log(message): - """ Print logging information - Args: - message (str): message to be printed - """ - - print('[%s]: %s' % (sys.argv[0], message)) - -def test_log(message): - """ Print logging information only if testing mode is enabled - Args: - message (str): message to be printed - """ - if testing: - print('[%s]: %s' % (sys.argv[0], message)) - -def copy_files(source_dir, target_dir): - """ Copy any files in the source_dir to the target_dir. - The copy is not recursive. - The directories must already exist. - Args: - source_dir (str): source directory path - target_dir (str): target directory path - Returns: - Nothing - """ - - global testing - assert os.path.isdir(source_dir) - assert testing or os.path.isdir(target_dir) - - for source_filename in os.listdir(source_dir): - source_pathname = os.path.join(source_dir, source_filename) - if os.path.isfile(source_pathname): - target_pathname = os.path.join(target_dir, source_filename) - log('Copy: %s => %s' % (source_pathname, target_pathname)) - if not testing: - shutil.copy2(source_pathname, target_pathname) - -def patch_coreclr_root(core_root, fx_bin): - """ Walk through the fx bin and patch corefx dlls to the core root. - Args: - core_root (str): the core root path - fx_bin (str): the runtime folder from a corefx build - Returns: - nothing - """ - test_log('Patching coreclr core_root') - - forbidden_names = ['coreclr.dll', - 'system.private.corelib.dll', - 'r2rdump.dll', - 'runincontext.dll', - 'mscordaccore.dll', - 'linuxnonjit.dll', - 'protononjit.dll', - 'mscordbi.dll', - 'clrjit.dll', - 'dbgshim.dll', - 'coreshim.dll', - 'clrgc.dll', - 'superpmi-shim-counter.dll', - 'clretwrc.dll', - 'superpmi-shim-collector.dll', - 'superpmi-shim-simple.dll', - 'jitinterface.dll', - 'mscorrc.debug.dll', - 'mscorrc.dll', - 'sos.dll'] - - test_log('forbidden_names = %s' % forbidden_names) - - for file in os.listdir(fx_bin): - test_log('considering file %s' % file) - - filename = os.path.basename(file) - comparename = filename.lower() - if ( comparename.endswith('.dll') and - comparename not in forbidden_names and - not comparename.startswith('api-ms-core') ): - source_pathname = os.path.join(fx_bin, filename) - target_pathname = os.path.join(core_root, filename) - - test_log ('copying file %s to file %s' % (source_pathname, target_pathname)) - - shutil.copy2(source_pathname, target_pathname) - -########################################################################## -# Main -########################################################################## - -def main(args): - """ - The way this script decides what to patch is by looking at the core - root for a list of dlls, then filtering out any ones built by coreclr. - This leaves us with a list of non-coreclr build dlls. Now we can use - that list to go through the corefx repo and any ones that also exist - in the corefx bin folder are copied over. - """ - - log('Patching CoreFX binaries from local enlistment.') - - arch, build_type, clr_core_root, fx_root = validate_args(args) - - clr_os = 'Windows_NT' if Is_windows else Unix_name_map[os.uname()[0]] - - if not os.path.exists(clr_core_root): - raise Exception('Core root path %s does not exist.' % (core_root)) - - fx_bin = os.path.join(fx_root, - 'artifacts', - 'bin', - 'runtime', - 'netcoreapp-%s-%s-%s' % (clr_os, build_type, arch)) - - if not os.path.exists(fx_bin): - raise Exception('CoreFX bin path %s does not exist.' % (core_root)) - - patch_coreclr_root(clr_core_root, fx_bin) - - -########################################################################## -# setup for Main -########################################################################## - -if __name__ == '__main__': - Args = parser.parse_args(sys.argv[1:]) - - main(Args) diff --git a/src/coreclr/tests/src/Common/CoreFX/CoreFX.csproj b/src/coreclr/tests/src/Common/CoreFX/CoreFX.csproj deleted file mode 100644 index 36dccb97ada493..00000000000000 --- a/src/coreclr/tests/src/Common/CoreFX/CoreFX.csproj +++ /dev/null @@ -1,150 +0,0 @@ - - - - - - $(SourceDir)Common\CoreFX\obj - C# - netcoreapp3.0 - SharedLibrary - false - 1.0.5 - Library - true - Microsoft.NETCore.App.deps.json - true - false - .NETCoreApp,Version=v3.0 - netcoreapp3.0 - true - $(MicrosoftNETCoreAppVersion) - - - - - win-$(Platform) - linux-$(Platform) - osx-$(Platform) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - hostfxr - dll - dotnet.exe - hostpolicy - dll - - - - libhostfxr - so - dylib - libhostpolicy - $(HostFxrFileExtension) - dotnet - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - $(SourceDir)Common\CoreFX\obj\project.assets.json - - - - - - - - - - - diff --git a/src/coreclr/tests/src/dirs.proj b/src/coreclr/tests/src/dirs.proj index c545e1c16140ec..617f1328a32a46 100644 --- a/src/coreclr/tests/src/dirs.proj +++ b/src/coreclr/tests/src/dirs.proj @@ -8,7 +8,7 @@ - + @@ -18,7 +18,6 @@ - @@ -125,7 +124,7 @@ - + diff --git a/src/coreclr/tests/src/runtest.proj b/src/coreclr/tests/src/runtest.proj index 60b711b12de38c..3dd4a20eb0be2b 100644 --- a/src/coreclr/tests/src/runtest.proj +++ b/src/coreclr/tests/src/runtest.proj @@ -376,16 +376,6 @@ namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\",""). Properties="Language=C#;TargetRid=$(TargetRid);RuntimeIdentifier=$(TargetRid)" /> - - - - - - @@ -404,10 +394,6 @@ namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\",""). Targets="CreateTestOverlay" Condition=" '$(GenerateRuntimeLayout)'=='true' "/> - - diff --git a/src/libraries/restore/runtime/runtime.depproj b/src/libraries/restore/runtime/runtime.depproj index 6c6b60bdf7d791..81abb31afd858a 100644 --- a/src/libraries/restore/runtime/runtime.depproj +++ b/src/libraries/restore/runtime/runtime.depproj @@ -13,11 +13,6 @@ - - - 4.6.0-rc2.19462.14 - - @@ -30,7 +25,7 @@ - + From cba76e8394506928ad2d31159d44dcbbc07c48c7 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 18 Nov 2019 15:08:29 -0800 Subject: [PATCH 15/71] Only copy the UCRT on Windows. --- src/coreclr/tests/src/Common/test_runtime/test_runtime.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/tests/src/Common/test_runtime/test_runtime.csproj b/src/coreclr/tests/src/Common/test_runtime/test_runtime.csproj index 6b35e3498a32ac..75eb52d392e7bf 100644 --- a/src/coreclr/tests/src/Common/test_runtime/test_runtime.csproj +++ b/src/coreclr/tests/src/Common/test_runtime/test_runtime.csproj @@ -17,7 +17,7 @@ - + From 3d3f216c753cf5a755a92cd71b573fb8350a90fd Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 19 Nov 2019 11:53:53 -0800 Subject: [PATCH 16/71] CoreCLR test build now builds against a live libraries sfx+oob ref assembly set. --- eng/liveBuilds.targets | 15 +++++--- .../test_dependencies.csproj | 1 - src/coreclr/tests/src/Directory.Build.targets | 34 +++++++++++++++---- src/libraries/Directory.Build.targets | 3 ++ 4 files changed, 41 insertions(+), 12 deletions(-) diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets index e6db37b34638e2..b2cb9a88edbb3d 100644 --- a/eng/liveBuilds.targets +++ b/eng/liveBuilds.targets @@ -5,12 +5,14 @@ $(Configuration) $(Configuration) $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'coreclr', '$(OSGroup).$(TargetArchitecture).$(CoreCLRConfiguration)')) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'ref', 'microsoft.netcore.app', '$(LibrariesConfiguration)')) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'ref', 'microsoft.netcore.app', '$(LibrariesConfiguration)')) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'ref', 'oob', '$(LibrariesConfiguration)')) $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'pkg', 'netcoreapp', 'runtime', '$(OSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'pkg', 'oob', 'runtime', '$(OSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'native', 'netcoreapp-$(OSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) $(CoreCLRArtifactsPathInTree) - $(LibrariesRefArtifactsPathInTree) + $(LibrariesSharedFrameworkRefArtifactsPathInTree) + $(LibrariesOOBRefArtifactsPathInTree) $(LibrariesSharedFrameworkBinArtifactsPathInTree) $(LibrariesOOBBinArtifactsPathInTree) $(LibrariesNativeArtifactsPathInTree) @@ -73,7 +75,8 @@ - true + true + true true true true @@ -90,12 +93,14 @@ OSGroup=$(OSGroup)" /> - + + - + + true win-arm;win-arm64;win-x64;win-x86;$(TargetRid) $(BuildArch) - $(BuildOS) false false true diff --git a/src/coreclr/tests/src/Directory.Build.targets b/src/coreclr/tests/src/Directory.Build.targets index 275d70b85fd767..e85c13f9dce8aa 100644 --- a/src/coreclr/tests/src/Directory.Build.targets +++ b/src/coreclr/tests/src/Directory.Build.targets @@ -44,8 +44,8 @@ - - + + <_CLRTestNeedsProjectToRun>false <_CLRTestNeedsProjectToRun Condition=" '$(_CLRTestNeedsToRun)' and '!$(_CLRTestBuildsExecutable)' ">true @@ -80,10 +80,10 @@ <_CopyNativeProjectBinaries Condition="'$(_WillCLRTestProjectBuild)' != 'true'">false <_CopyNativeProjectBinaries Condition="'$(_CopyNativeProjectBinaries)' == ''">true - + - + @@ -105,7 +105,7 @@ - @@ -199,6 +199,12 @@ + + + + false @@ -207,6 +213,13 @@ + + + + + + - + true + + false + false + true + $(BuildArch) + + + + diff --git a/src/libraries/Directory.Build.targets b/src/libraries/Directory.Build.targets index 4795ac3ec2c438..c61584d9fdda47 100644 --- a/src/libraries/Directory.Build.targets +++ b/src/libraries/Directory.Build.targets @@ -81,6 +81,9 @@ + + $(RefRootPath)oob\$(ConfigurationGroup) + $(ArtifactsBinDir)pkg\oob\runtime\$(_bc_OSGroup)-$(ConfigurationGroup)-$(ArchGroup) From 2f9d951d923ccbbecb2d02ff18e51bef16bc3eb0 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 19 Nov 2019 12:54:23 -0800 Subject: [PATCH 17/71] Override GetCorePackagePaths target to not reference unused packages (breaks the live-live build on a clean system). --- .../src/localnetcoreapp.override.targets | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/installer/pkg/projects/netcoreapp/src/localnetcoreapp.override.targets b/src/installer/pkg/projects/netcoreapp/src/localnetcoreapp.override.targets index 0e6ae6b8c293fc..5d86576c26425a 100644 --- a/src/installer/pkg/projects/netcoreapp/src/localnetcoreapp.override.targets +++ b/src/installer/pkg/projects/netcoreapp/src/localnetcoreapp.override.targets @@ -1,4 +1,36 @@ + + + + <_winmdPackageDir>$(NuGetPackageRoot)$(MicrosoftTargetingPackPrivateWinRTPackage.ToLowerInvariant())/$(MicrosoftTargetingPackPrivateWinRTVersion)/ + <_diaSymReaderPackageDir>$(NuGetPackageRoot)microsoft.diasymreader.native/$(MicrosoftDiaSymReaderNativeVersion)/ + + + + <_crossHostArch>x86 + + + <_crossHostArch>x64 + + + <_crossHostArch>x64 + + + <_crossDir Condition="'$(_crossHostArch)' != ''">/$(_crossHostArch)_$(TargetArchitecture) + + + + <_requiredProperty Include="_winmdPackageDir" /> + + + + + + + From 6601e93dea12fd48144f1fc99270b2b6a4ee6668 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 19 Nov 2019 13:06:03 -0800 Subject: [PATCH 18/71] Remove live references to Microsoft.NETCore.Platforms now that it seems to be unneeded since the SDK now packages a RID graph. Use the live RID graph in test_runtime.csproj where we may actually be looking for the live RID graph. --- eng/Tools.props | 5 +---- eng/Version.Details.xml | 4 ---- eng/liveBuilds.targets | 5 +++++ .../tests/src/Common/test_runtime/test_runtime.csproj | 2 +- .../pkg/projects/netcoreapp/src/netcoreapp.depproj | 7 +------ src/libraries/restore/runtime/runtime.depproj | 1 - 6 files changed, 8 insertions(+), 16 deletions(-) diff --git a/eng/Tools.props b/eng/Tools.props index a763640c26f89b..4d7d628472d50f 100644 --- a/eng/Tools.props +++ b/eng/Tools.props @@ -12,14 +12,11 @@ - + - - - diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 79cc85afbad13c..b4cb56270892f8 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,9 +1,5 @@ - - https://github.com/dotnet/corefx - 5cee7c97d602f294e27c582d4dab81ec388f1d7b - https://github.com/dotnet/corefx 5cee7c97d602f294e27c582d4dab81ec388f1d7b diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets index b2cb9a88edbb3d..94638b6bf87193 100644 --- a/eng/liveBuilds.targets +++ b/eng/liveBuilds.targets @@ -122,4 +122,9 @@ + + + $([MSBuild]::NormalizePath('$(LibrariesProjectRoot)', 'pkg', 'Microsoft.NETCore.Platforms', 'runtime.json')) + $(LiveRuntimeIdentifierGraphPath) + diff --git a/src/coreclr/tests/src/Common/test_runtime/test_runtime.csproj b/src/coreclr/tests/src/Common/test_runtime/test_runtime.csproj index 75eb52d392e7bf..abf800e5f2d4ec 100644 --- a/src/coreclr/tests/src/Common/test_runtime/test_runtime.csproj +++ b/src/coreclr/tests/src/Common/test_runtime/test_runtime.csproj @@ -9,9 +9,9 @@ $(NoWarn);NU1603 $(NoWarn);NU1701 true + true - diff --git a/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj b/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj index 8e8506513bf4ea..e90d45ca36f979 100644 --- a/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj +++ b/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj @@ -7,8 +7,6 @@ - - @@ -18,11 +16,8 @@ - + - - false diff --git a/src/coreclr/src/.nuget/packaging.props b/src/coreclr/src/.nuget/packaging.props index 6d16a9848935cd..69365bcd15d35b 100644 --- a/src/coreclr/src/.nuget/packaging.props +++ b/src/coreclr/src/.nuget/packaging.props @@ -6,7 +6,6 @@ $(CoreclrDir)/LICENSE.TXT $(CoreclrDir)/THIRD-PARTY-NOTICES.TXT - $(NuGetPackageRoot)/microsoft.netcore.platforms/$(MicrosoftNETCorePlatformsVersion)/runtime.json https://go.microsoft.com/fwlink/?LinkID=799421 https://dot.net diff --git a/src/installer/pkg/Directory.Build.targets b/src/installer/pkg/Directory.Build.targets index cbe076d1e7f617..d457972bfd7f5a 100644 --- a/src/installer/pkg/Directory.Build.targets +++ b/src/installer/pkg/Directory.Build.targets @@ -1,10 +1,6 @@ - - $(NuGetPackageRoot)$(PlatformPackageId.ToLowerInvariant())\$(MicrosoftNETCorePlatformsVersion)\runtime.json - - From 0344a538d6417879d2918085a5b088780c650e83 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 19 Nov 2019 14:21:26 -0800 Subject: [PATCH 21/71] Fix problems in installer build in live-live that were found from a clean build. --- .../netcoreapp/src/localnetcoreapp.override.targets | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/installer/pkg/projects/netcoreapp/src/localnetcoreapp.override.targets b/src/installer/pkg/projects/netcoreapp/src/localnetcoreapp.override.targets index 5d86576c26425a..d6fe8055204f31 100644 --- a/src/installer/pkg/projects/netcoreapp/src/localnetcoreapp.override.targets +++ b/src/installer/pkg/projects/netcoreapp/src/localnetcoreapp.override.targets @@ -35,9 +35,6 @@ DependsOnTargets="GetCorePackagePaths"> - <_runtimeCLR Include="$(_runtimePackageDir)**/$(LibraryFilePrefix)coreclr$(LibraryFileExtension)" /> - <_runtimeCoreLib Include="$(_runtimePackageDir)**/native/System.Private.CoreLib.dll" /> - <_fxSystemRuntime Include="$(_corefxPackageDir)**/System.Runtime.dll" /> <_windowsWinMD Include="$(_winmdPackageDir)**/Windows.winmd" /> <_diaSymReaderAssembly Include="$(_diaSymReaderPackageDir)**\Microsoft.DiaSymReader.Native.*.dll" /> @@ -47,18 +44,15 @@ <_crossGenPath>$([MSBuild]::NormalizePath('$(CoreCLRArtifactsPath)', '$(CoreCLRCrossTargetComponentDirName)', 'sharedFramework', 'crossgen$(ApplicationFileExtension)')) - + <_coreLibDirectory>$(CoreCLRArtifactsPath) + <_fxLibDirectory>$(LibrariesSharedFrameworkBinArtifactsPath) <_jitPath>$([MSBuild]::NormalizePath('$(CoreCLRArtifactsPath)', '$(CoreCLRCrossTargetComponentDirName)', 'sharedFramework', '$(LibraryFilePrefix)clrjit$(LibraryFileExtension)')) - - <_fxLibDirectory>%(_fxSystemRuntime.RootDir)%(_fxSystemRuntime.Directory) - - <_windowsWinMDDirectory>%(_windowsWinMD.RootDir)%(_windowsWinMD.Directory) @@ -89,6 +83,7 @@ <_crossgenPlatformDirectories Include="$(_runtimeDirectory)" /> <_crossgenPlatformDirectories Include="$(_coreLibDirectory)" /> + <_crossgenPlatformDirectories Include="$(_fxLibDirectory)" /> From 314c4a01c43dc5bcf524d2b767ef726550de42eb Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 19 Nov 2019 14:47:00 -0800 Subject: [PATCH 22/71] Remove unused dependency on Microsoft.NETCore.Targets. --- eng/Version.Details.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 78a4ee731f135e..394f24ca525bca 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -4,10 +4,6 @@ https://github.com/dotnet/corefx 5cee7c97d602f294e27c582d4dab81ec388f1d7b - - https://github.com/dotnet/corefx - 5cee7c97d602f294e27c582d4dab81ec388f1d7b - https://github.com/dotnet/corefx 5cee7c97d602f294e27c582d4dab81ec388f1d7b From f640bd9749452ee39b92e29b6256e6c25d997f8e Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 19 Nov 2019 14:57:35 -0800 Subject: [PATCH 23/71] Fix Microsoft.NETCore.App -> Microsoft.NETCore.Platforms dependency. --- .../netcoreapp/pkg/workaround/Microsoft.NETCore.App.pkgproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/installer/pkg/projects/netcoreapp/pkg/workaround/Microsoft.NETCore.App.pkgproj b/src/installer/pkg/projects/netcoreapp/pkg/workaround/Microsoft.NETCore.App.pkgproj index 5a0d9501e728d9..d9d6f3423c3dda 100644 --- a/src/installer/pkg/projects/netcoreapp/pkg/workaround/Microsoft.NETCore.App.pkgproj +++ b/src/installer/pkg/projects/netcoreapp/pkg/workaround/Microsoft.NETCore.App.pkgproj @@ -20,7 +20,7 @@ - + From 80669fab32990a7bf1ed9941baac99e8930eed72 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 19 Nov 2019 14:58:27 -0800 Subject: [PATCH 24/71] Remove Microsoft.NETCore.Targets version from Versions.props. --- eng/Versions.props | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 381d8a0f3b9d2e..9191ebbf4a0d70 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -53,7 +53,6 @@ 5.0.0-alpha1.19563.3 5.0.0-alpha.1.19563.6 - 5.0.0-alpha.1.19563.6 5.0.0-alpha.1.19563.6 5.0.0-alpha.1.19563.6 5.0.0-alpha.1.19563.3 From cea542d4a294650cb8ad72f00d9f3bfb8728d9b6 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 19 Nov 2019 15:11:57 -0800 Subject: [PATCH 25/71] Remove unused deps.csproj file that was the last dependency on the packaged Microsoft.NETCore.Platforms package. --- eng/Version.Details.xml | 4 ---- eng/Versions.props | 2 -- src/installer/pkg/deps/deps.csproj | 11 ----------- 3 files changed, 17 deletions(-) delete mode 100644 src/installer/pkg/deps/deps.csproj diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 394f24ca525bca..763bf68e758792 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,9 +1,5 @@ - - https://github.com/dotnet/corefx - 5cee7c97d602f294e27c582d4dab81ec388f1d7b - https://github.com/dotnet/corefx 5cee7c97d602f294e27c582d4dab81ec388f1d7b diff --git a/eng/Versions.props b/eng/Versions.props index 9191ebbf4a0d70..dac7017e8d5532 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -52,7 +52,6 @@ 5.0.0-alpha1.19563.3 5.0.0-alpha1.19563.3 - 5.0.0-alpha.1.19563.6 5.0.0-alpha.1.19563.6 5.0.0-alpha.1.19563.6 5.0.0-alpha.1.19563.3 @@ -104,7 +103,6 @@ optimization.linux-x64.ibc.corefx microsoft.private.intellisense Microsoft.DotNet.XUnitConsoleRunner - Microsoft.NETCore.Platforms Microsoft.NETCore.App optimization.IBC.CoreCLR optimization.PGO.CoreCLR diff --git a/src/installer/pkg/deps/deps.csproj b/src/installer/pkg/deps/deps.csproj deleted file mode 100644 index f105b8f3826f70..00000000000000 --- a/src/installer/pkg/deps/deps.csproj +++ /dev/null @@ -1,11 +0,0 @@ - - - - netstandard1.0 - - - - - - - From 425d0347dd96b83c3bc839511c75c5c6403dc4d9 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 20 Nov 2019 09:28:57 -0800 Subject: [PATCH 26/71] Fix OSGroup calculation for dependent projects off-Windows. --- eng/liveBuilds.targets | 21 ++++++++++++------- .../netcoreapp/src/netcoreapp.depproj | 5 +++++ src/libraries/restore/runtime/runtime.depproj | 1 + 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets index 1f9cde3b8e406a..b3b8688ca20019 100644 --- a/eng/liveBuilds.targets +++ b/eng/liveBuilds.targets @@ -2,14 +2,16 @@ $([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant) + $(OSGroup) $(Configuration) + $(OSGroup) $(Configuration) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'coreclr', '$(OSGroup).$(TargetArchitecture).$(CoreCLRConfiguration)')) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'coreclr', '$(CoreCLROSGroup).$(TargetArchitecture).$(CoreCLRConfiguration)')) $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'ref', 'microsoft.netcore.app', '$(LibrariesConfiguration)')) $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'ref', 'oob', '$(LibrariesConfiguration)')) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'pkg', 'netcoreapp', 'runtime', '$(OSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'pkg', 'oob', 'runtime', '$(OSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'native', 'netcoreapp-$(OSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'pkg', 'netcoreapp', 'runtime', '$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'pkg', 'oob', 'runtime', '$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'native', 'netcoreapp-$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) $(CoreCLRArtifactsPathInTree) $(LibrariesSharedFrameworkRefArtifactsPathInTree) $(LibrariesOOBRefArtifactsPathInTree) @@ -32,9 +34,10 @@ Targets="Build" BuildInParallel="true" Properties="Configuration=$(CoreCLRConfiguration); - TargetArchitecture=$(TargetArchitecture)" /> + TargetArchitecture=$(TargetArchitecture); + OSGroup=$(CoreCLROSGroup)" /> - + $([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)')) @@ -58,7 +61,9 @@ $(CoreCLRSharedFrameworkDir)PDB/*.dbg; $(CoreCLRSharedFrameworkDir)PDB/*.dwarf" /> + Include="$(CoreCLRArtifactsPath)PDB/System.Private.CoreLib.pdb;" /> + + OSGroup=$(LibrariesOSGroup)" /> diff --git a/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj b/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj index e90d45ca36f979..b06c97666f80cb 100644 --- a/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj +++ b/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj @@ -4,6 +4,11 @@ Microsoft.NETCore.App true false + Windows_NT + Linux + OSX + FreeBSD + $(CoreCLROSGroup) diff --git a/src/libraries/restore/runtime/runtime.depproj b/src/libraries/restore/runtime/runtime.depproj index d2d097ce362c2b..d6dcba5cafb16f 100644 --- a/src/libraries/restore/runtime/runtime.depproj +++ b/src/libraries/restore/runtime/runtime.depproj @@ -6,6 +6,7 @@ $(NoWarn);NU1603;NU1605 true true + $(DefaultOSGroup) $(ConfigurationGroup) From 364c6213eb008d596a1cc0189f7ce51ecadb5617 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 20 Nov 2019 15:22:45 -0800 Subject: [PATCH 27/71] Fix construction of Microsoft.NETCore.App.Internal package. --- .../pkg/legacy/Microsoft.NETCore.App.Internal.pkgproj | 5 +++++ src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/installer/pkg/projects/netcoreapp/pkg/legacy/Microsoft.NETCore.App.Internal.pkgproj b/src/installer/pkg/projects/netcoreapp/pkg/legacy/Microsoft.NETCore.App.Internal.pkgproj index 8a92a5716b84c0..bea575e85c83ba 100644 --- a/src/installer/pkg/projects/netcoreapp/pkg/legacy/Microsoft.NETCore.App.Internal.pkgproj +++ b/src/installer/pkg/projects/netcoreapp/pkg/legacy/Microsoft.NETCore.App.Internal.pkgproj @@ -18,5 +18,10 @@ false + + + + + diff --git a/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj b/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj index b06c97666f80cb..ced5fb1500f081 100644 --- a/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj +++ b/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj @@ -43,7 +43,7 @@ - + From 002e6640b63debdab2e7b5dbcaad140c97deda9e Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 21 Nov 2019 08:58:25 -0800 Subject: [PATCH 28/71] Remove unneeded CoreRun copying. --- src/libraries/restore/runtime/runtime.depproj | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/libraries/restore/runtime/runtime.depproj b/src/libraries/restore/runtime/runtime.depproj index d6dcba5cafb16f..1603fbf2374bd8 100644 --- a/src/libraries/restore/runtime/runtime.depproj +++ b/src/libraries/restore/runtime/runtime.depproj @@ -72,12 +72,8 @@ DependsOnTargets="ResolveCoreCLRFilesFromLocalBuild" AfterTargets="AfterResolveReferences;FilterNugetPackages"> - - - - From e510223d18ac619f3293d9e0e914872d51f8ace2 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 3 Dec 2019 13:32:28 -0800 Subject: [PATCH 29/71] Serialize subsets that have dependencies across subset groups. Don't use `` tasks to rebuild subsets since we don't want to rebuild projects multiple times (coreclr was being built multiple times). --- eng/Subsets.props | 26 ++++++++--------- eng/liveBuilds.targets | 50 ++++----------------------------- src/libraries/restore/dirs.proj | 1 - 3 files changed, 19 insertions(+), 58 deletions(-) diff --git a/eng/Subsets.props b/eng/Subsets.props index d6cedf32602aac..e020dea1900726 100644 --- a/eng/Subsets.props +++ b/eng/Subsets.props @@ -99,18 +99,18 @@ - - - - - - - + + + + + + + @@ -124,23 +124,23 @@ - + - + - + - - + + @@ -152,7 +152,7 @@ - + diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets index b3b8688ca20019..e0f7daeb068c62 100644 --- a/eng/liveBuilds.targets +++ b/eng/liveBuilds.targets @@ -6,37 +6,18 @@ $(Configuration) $(OSGroup) $(Configuration) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'coreclr', '$(CoreCLROSGroup).$(TargetArchitecture).$(CoreCLRConfiguration)')) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'ref', 'microsoft.netcore.app', '$(LibrariesConfiguration)')) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'ref', 'oob', '$(LibrariesConfiguration)')) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'pkg', 'netcoreapp', 'runtime', '$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'pkg', 'oob', 'runtime', '$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'native', 'netcoreapp-$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) - $(CoreCLRArtifactsPathInTree) - $(LibrariesSharedFrameworkRefArtifactsPathInTree) - $(LibrariesOOBRefArtifactsPathInTree) - $(LibrariesSharedFrameworkBinArtifactsPathInTree) - $(LibrariesOOBBinArtifactsPathInTree) - $(LibrariesNativeArtifactsPathInTree) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'coreclr', '$(CoreCLROSGroup).$(TargetArchitecture).$(CoreCLRConfiguration)')) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'ref', 'microsoft.netcore.app', '$(LibrariesConfiguration)')) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'ref', 'oob', '$(LibrariesConfiguration)')) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'pkg', 'netcoreapp', 'runtime', '$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'pkg', 'oob', 'runtime', '$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'native', 'netcoreapp-$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) x64 x86 x64 - - true - - - - @@ -79,25 +60,6 @@ - - true - true - true - true - true - - - - - diff --git a/src/libraries/restore/dirs.proj b/src/libraries/restore/dirs.proj index cf1b99f6e1e232..9c11cf6e8230e5 100644 --- a/src/libraries/restore/dirs.proj +++ b/src/libraries/restore/dirs.proj @@ -12,7 +12,6 @@ - From 5e03d58fa6b43ac06c4c5c71d0ed1434886afed7 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 3 Dec 2019 13:39:07 -0800 Subject: [PATCH 30/71] Cleanup from PR feedback. --- eng/liveBuilds.targets | 2 -- src/coreclr/tests/src/Directory.Build.targets | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets index e0f7daeb068c62..17de67f59708d4 100644 --- a/eng/liveBuilds.targets +++ b/eng/liveBuilds.targets @@ -22,8 +22,6 @@ $([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)')) - - $([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)','sharedFramework')) $([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)','$(CoreCLRCrossTargetComponentDir)','sharedFramework')) diff --git a/src/coreclr/tests/src/Directory.Build.targets b/src/coreclr/tests/src/Directory.Build.targets index e85c13f9dce8aa..cee75dcaf10e3b 100644 --- a/src/coreclr/tests/src/Directory.Build.targets +++ b/src/coreclr/tests/src/Directory.Build.targets @@ -234,13 +234,11 @@ - + true - false - false true $(BuildArch) From dc3916ff13a4384656876a88257c2017236b9d4b Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 3 Dec 2019 16:30:19 -0800 Subject: [PATCH 31/71] Only copy libraries outputs to artifacts/bin/runtime. --- src/libraries/Directory.Build.targets | 3 ++- src/libraries/shims/Directory.Build.props | 1 + src/libraries/shims/manual/Directory.Build.props | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/libraries/Directory.Build.targets b/src/libraries/Directory.Build.targets index 164d9be318aac6..32bde4a179e4ee 100644 --- a/src/libraries/Directory.Build.targets +++ b/src/libraries/Directory.Build.targets @@ -44,7 +44,8 @@ - + + $(BuildConfigurationRefPath) $(RuntimePath) diff --git a/src/libraries/shims/Directory.Build.props b/src/libraries/shims/Directory.Build.props index d496b6b11b83cc..c3b4741dca4261 100644 --- a/src/libraries/shims/Directory.Build.props +++ b/src/libraries/shims/Directory.Build.props @@ -21,6 +21,7 @@ true false true + true diff --git a/src/libraries/shims/manual/Directory.Build.props b/src/libraries/shims/manual/Directory.Build.props index 57cdeb82eef669..fa1ef91cdfb7ab 100644 --- a/src/libraries/shims/manual/Directory.Build.props +++ b/src/libraries/shims/manual/Directory.Build.props @@ -3,7 +3,7 @@ manual.$(MSBuildProjectName) - + @@ -19,5 +19,6 @@ + From 605322d18eec8ab3acc9952206e7f7552c2dc67e Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 3 Dec 2019 16:42:38 -0800 Subject: [PATCH 32/71] Use the netcoreapp output directories for OOB-included references to libraries outputs. --- eng/liveBuilds.targets | 18 +++++++++--------- src/coreclr/build-test.cmd | 16 ---------------- src/libraries/Directory.Build.targets | 8 -------- 3 files changed, 9 insertions(+), 33 deletions(-) diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets index 17de67f59708d4..938c72502e1035 100644 --- a/eng/liveBuilds.targets +++ b/eng/liveBuilds.targets @@ -8,9 +8,9 @@ $(Configuration) $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'coreclr', '$(CoreCLROSGroup).$(TargetArchitecture).$(CoreCLRConfiguration)')) $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'ref', 'microsoft.netcore.app', '$(LibrariesConfiguration)')) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'ref', 'oob', '$(LibrariesConfiguration)')) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'ref', 'netcoreapp')) $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'pkg', 'netcoreapp', 'runtime', '$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'pkg', 'oob', 'runtime', '$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'runtime', 'netcoreapp-$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'native', 'netcoreapp-$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) x64 x86 @@ -59,20 +59,20 @@ - + - + - - - + + ILLinkTrimAssembly=true - - - $(RefRootPath)oob\$(ConfigurationGroup) - - - $(ArtifactsBinDir)pkg\oob\runtime\$(_bc_OSGroup)-$(ConfigurationGroup)-$(ArchGroup) - - $(NETCoreAppTestSharedFrameworkPath) From 85e5f6b30ddc51b749bf9289b4f1e5c5c6c53b33 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 3 Dec 2019 16:48:23 -0800 Subject: [PATCH 33/71] Remove references to deleted old corefx override setup. --- src/coreclr/build-test.cmd | 3 --- src/coreclr/build-test.sh | 1 - 2 files changed, 4 deletions(-) diff --git a/src/coreclr/build-test.cmd b/src/coreclr/build-test.cmd index 06b319fc800876..f2891f3753c107 100644 --- a/src/coreclr/build-test.cmd +++ b/src/coreclr/build-test.cmd @@ -59,7 +59,6 @@ set __DoCrossgen2= set __CopyNativeTestBinaries=0 set __CopyNativeProjectsAfterCombinedTestBuild=true set __SkipGenerateLayout=0 -set __LocalCoreFXPath= set __LocalCoreFXConfig=%__BuildType% set __SkipFXRestoreArg= set __GenerateLayoutOnly=0 @@ -106,8 +105,6 @@ if /i "%1" == "Exclude" (set __Exclude=%2&set processedArgs=!proce if /i "%1" == "-priority" (set __Priority=%2&shift&set processedArgs=!processedArgs! %1=%2&shift&goto Arg_Loop) if /i "%1" == "copynativeonly" (set __CopyNativeTestBinaries=1&set __SkipNative=1&set __CopyNativeProjectsAfterCombinedTestBuild=false&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "skipgeneratelayout" (set __SkipGenerateLayout=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) -if /i "%1" == "localcorefxpath" (set __LocalCoreFXPath=%2&set __SkipFXRestoreArg=/p:__SkipFXRestore=true&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) -if /i "%1" == "localcorefxconfig" (set __LocalCoreFXConfig=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) if /i "%1" == "generatelayoutonly" (set __SkipManaged=1&set __SkipNative=1&set __CopyNativeProjectsAfterCombinedTestBuild=false&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "--" (set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) diff --git a/src/coreclr/build-test.sh b/src/coreclr/build-test.sh index 86f0b8123d7c0c..2a87db5fe2674d 100755 --- a/src/coreclr/build-test.sh +++ b/src/coreclr/build-test.sh @@ -645,7 +645,6 @@ __SkipRestore="" __SkipRestorePackages=0 __SourceDir="$__ProjectDir/src" __UnprocessedBuildArgs= -__LocalCoreFXPath= __LocalCoreFXConfig=${__BuildType} __UseNinja=0 __VerboseBuild=0 From 8ee14f54ff21dc8448670a93f17182d81f805f66 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 3 Dec 2019 16:49:30 -0800 Subject: [PATCH 34/71] First pass to get new runtime pipeline using the local live-live infra (coreclr+libraries only for now). --- .../coreclr/templates/xplat-pipeline-job.yml | 11 ++++------- eng/pipelines/libraries/base-job.yml | 6 +++--- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index b36cad5bf86bce..9f1876a86de7d2 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -41,7 +41,7 @@ jobs: stagedBuild: ${{ parameters.stagedBuild }} strategy: ${{ parameters.strategy }} pool: ${{ parameters.pool }} - + # arcade-specific parameters condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} @@ -50,7 +50,7 @@ jobs: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} enableMicrobuild: ${{ parameters.enableMicrobuild }} gatherAssetManifests: ${{ parameters.gatherAssetManifests }} - + variables: - ${{ if ne(parameters.testGroup, '') }}: - name: testArtifactRootName @@ -89,7 +89,7 @@ jobs: - name: microsoftNetSdkIlFolderPath value: '$(Build.SourcesDirectory)/.packages/microsoft.net.sdk.il' - + - name: microsoftNetSdkIlArtifactName value: 'MicrosoftNetSdkIlPackage_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_$(archType)_$(buildConfig)' @@ -111,10 +111,7 @@ jobs: - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - librariesArtifactName: ${{ format('libraries_bin_{0}_{1}{2}_{3}_{4}', 'netcoreapp', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} - librariesDownloadDir: $(Build.SourcesDirectory)/artifacts - - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - librariesOverrideArg: ' localcorefxpath $(Build.SourcesDirectory) localcorefxconfig ${{ parameters.liveLibrariesBuildConfig }}' - - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - librariesOverrideArg: ' localcorefxpath=$(Build.SourcesDirectory) localcorefxconfig=${{ parameters.liveLibrariesBuildConfig }}' + - librariesOverrideArg: ' /p:LibrariesConfiguration=${{ parameters.liveLibrariesBuildConfig }}' - ${{ each variable in parameters.variables }}: - ${{insert}}: ${{ variable }} diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index cceb842179cb12..2b8d83c8e9975e 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -50,10 +50,10 @@ jobs: - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubGroup, '_rhel6')) }}: - _runtimeOSArg: /p:RuntimeOS=rhel.6 - _portableArg: /p:PortableBuild=false - + - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubGroup, '_musl')) }}: - _runtimeOSArg: /p:RuntimeOS=linux-musl - + - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubGroup, ''), eq(parameters.archType, 'arm')) }}: - _runtimeOSArg: /p:RuntimeOS=ubuntu.16.04 @@ -76,7 +76,7 @@ jobs: - ${{ if ne(parameters.liveCoreClrBuildConfig, '') }}: - _coreClrDownloadPath: '$(Build.SourcesDirectory)/artifacts/transport/coreclr' - _coreClrArtifactName: 'CoreCLRProduct_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveCoreClrBuildConfig }}' - - _coreClrOverridePathArg: ' /p:CoreCLROverridePath=$(_coreClrDownloadPath)' + - _coreClrOverridePathArg: ' /p:CoreCLRArtifactsPath=$(_coreClrDownloadPath)' # Windows variables - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: From dfa1ec60f2e31e6b19abf7e929205f520c934a88 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 3 Dec 2019 17:19:32 -0800 Subject: [PATCH 35/71] Don't generate layout during test native build. --- eng/pipelines/coreclr/templates/build-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml index 86d33eb897fc25..cdf7436e423c36 100644 --- a/eng/pipelines/coreclr/templates/build-job.yml +++ b/eng/pipelines/coreclr/templates/build-job.yml @@ -142,7 +142,7 @@ jobs: displayName: Build product # Build native test components - - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipmanaged $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(clangArg) + - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipmanaged $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(clangArg) skipgeneratelayout displayName: Build native test components # Sign on Windows From e70bbc823bcaa502ae4405328415d9f055d7f382 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 4 Dec 2019 11:04:14 -0800 Subject: [PATCH 36/71] Fix ref assembly default binplacing for netcoreapp. --- src/libraries/Directory.Build.targets | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/Directory.Build.targets b/src/libraries/Directory.Build.targets index f5dd56fedd0ff5..a1fca0bf4df89d 100644 --- a/src/libraries/Directory.Build.targets +++ b/src/libraries/Directory.Build.targets @@ -44,8 +44,8 @@ - - + + $(BuildConfigurationRefPath) $(RuntimePath) From 374d270768cdd8a7b3f2f01095b6756a52945f22 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 4 Dec 2019 11:15:57 -0800 Subject: [PATCH 37/71] Remove msbuild goo around importing Subsets.props and Directory.Build.props only once each. This goo isn't needed anymore. --- Directory.Build.props | 10 ---------- eng/Subsets.props | 3 --- 2 files changed, 13 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index e42f9a152091fc..41205ba0921d1c 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -18,10 +18,6 @@ --> - - true - - @@ -50,12 +46,6 @@ $([MSBuild]::NormalizeDirectory('$(LibrariesProjectRoot)', 'System.Private.CoreLib', 'src')) - - - - true $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'libraries')) $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'coreclr')) $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'installer')) $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'tools-local')) $([MSBuild]::NormalizeDirectory('$(RepoToolsLocalDir)', 'tasks')) - - libraries-installer-coreclr From 14c9ea78b622e99e3d6e0a14090a7f66eae639f5 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 4 Dec 2019 12:29:06 -0800 Subject: [PATCH 38/71] Fix disabling of binplacing for the target vertical to exclude only runtime.depproj and not exclude winrt.depproj. --- src/libraries/Directory.Build.targets | 4 ++-- src/libraries/restore/runtime/runtime.depproj | 1 + src/libraries/shims/Directory.Build.props | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libraries/Directory.Build.targets b/src/libraries/Directory.Build.targets index a1fca0bf4df89d..91ba959e6ba7cb 100644 --- a/src/libraries/Directory.Build.targets +++ b/src/libraries/Directory.Build.targets @@ -40,12 +40,12 @@ true true + true - - + $(BuildConfigurationRefPath) $(RuntimePath) diff --git a/src/libraries/restore/runtime/runtime.depproj b/src/libraries/restore/runtime/runtime.depproj index 28a73401c1ed64..8055e8010d4da2 100644 --- a/src/libraries/restore/runtime/runtime.depproj +++ b/src/libraries/restore/runtime/runtime.depproj @@ -8,6 +8,7 @@ true $(DefaultOSGroup) $(ConfigurationGroup) + false diff --git a/src/libraries/shims/Directory.Build.props b/src/libraries/shims/Directory.Build.props index c3b4741dca4261..d496b6b11b83cc 100644 --- a/src/libraries/shims/Directory.Build.props +++ b/src/libraries/shims/Directory.Build.props @@ -21,7 +21,6 @@ true false true - true From c53137343c6edd4a4213b8b03d6723e8647b870c Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 5 Dec 2019 12:27:55 -0800 Subject: [PATCH 39/71] Fix libraries test build. --- eng/pipelines/libraries/build-job.yml | 14 ++++++++++---- eng/referenceFromRuntime.targets | 5 +++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index 9e4bf1cc4d1e58..8eb8c76bc10814 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -107,19 +107,25 @@ jobs: inputs: sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/runtime targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin/runtime - + + - task: CopyFiles@2 + displayName: Prepare external.runtime folder to publish + inputs: + sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/external.runtime + targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin/external.runtime + - task: CopyFiles@2 displayName: Prepare testhost folder to publish inputs: sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/testhost targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin/testhost - + - task: CopyFiles@2 displayName: Prepare artifacts toolset folder to publish inputs: sourceFolder: $(Build.SourcesDirectory)/artifacts/toolset targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/toolset - + - template: /eng/pipelines/common/upload-artifact-step.yml parameters: rootFolder: $(Build.ArtifactStagingDirectory)/artifacts @@ -129,7 +135,7 @@ jobs: tarCompression: $(tarCompression) artifactName: ${{ format('libraries_bin_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} displayName: Build Assets - + - ${{ if eq(parameters.runTests, true) }}: - template: /eng/pipelines/libraries/helix.yml parameters: diff --git a/eng/referenceFromRuntime.targets b/eng/referenceFromRuntime.targets index 327fab8cc47fa8..a5a6a42ebd92fc 100644 --- a/eng/referenceFromRuntime.targets +++ b/eng/referenceFromRuntime.targets @@ -3,10 +3,11 @@ BeforeTargets="SetupDefaultReferences" Condition="'$(IsTestProject)'=='true' and '@(ReferenceFromRuntime)' != ''"> - + + - + AddRuntimeProjectReference; From 6269367183493ea188e563f1397ba67ad90ed390 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 5 Dec 2019 12:34:59 -0800 Subject: [PATCH 40/71] Copy all paths used by the "ResolveLibrariesFromLocalBuild" target to publish from libraries build. --- eng/pipelines/libraries/build-job.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index 8eb8c76bc10814..0f85213bd40bf8 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -114,6 +114,24 @@ jobs: sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/external.runtime targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin/external.runtime + - task: CopyFiles@2 + displayName: Prepare shared framework ref assemblies to publish + inputs: + sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/ref/microsoft.netcore.app + targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin/ref/microsoft.netcore.app + + - task: CopyFiles@2 + displayName: Prepare shared framework implementation assemblies to publish + inputs: + sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/pkg/netcoreapp/runtime + targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin/pkg/netcoreapp/runtime + + - task: CopyFiles@2 + displayName: Prepare native folder to publish + inputs: + sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/native + targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin/native + - task: CopyFiles@2 displayName: Prepare testhost folder to publish inputs: From b9fe7cc2fdefb81e056bd94d04b89e769de15e42 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 5 Dec 2019 15:41:23 -0800 Subject: [PATCH 41/71] For shims, reference the System.Private.CoreLib directly from CoreCLR outputs instead of through runtime.depproj. --- src/libraries/shims/manual/Directory.Build.props | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/libraries/shims/manual/Directory.Build.props b/src/libraries/shims/manual/Directory.Build.props index fa1ef91cdfb7ab..c6c1f72b25758f 100644 --- a/src/libraries/shims/manual/Directory.Build.props +++ b/src/libraries/shims/manual/Directory.Build.props @@ -13,12 +13,20 @@ true $(ArtifactsBinDir)runtime/$(TargetGroup)-$(_bc_OSGroup)-$(_bc_ConfigurationGroup)-$(ArchGroup)/ + true + $(_bc_OSGroup) + $(_bc_ConfigurationGroup) - + + + + + + From d3617313c9a100df1a74f85d540f3dc5246b06c4 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 5 Dec 2019 15:55:00 -0800 Subject: [PATCH 42/71] Hard-code the libraries configuration to release in coreclr test builds. --- .../src/Common/test_dependencies/test_dependencies.csproj | 3 +-- src/coreclr/tests/src/Directory.Build.targets | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/tests/src/Common/test_dependencies/test_dependencies.csproj b/src/coreclr/tests/src/Common/test_dependencies/test_dependencies.csproj index 1771616ef70d6d..864e1322e1c446 100644 --- a/src/coreclr/tests/src/Common/test_dependencies/test_dependencies.csproj +++ b/src/coreclr/tests/src/Common/test_dependencies/test_dependencies.csproj @@ -7,9 +7,8 @@ true win-arm;win-arm64;win-x64;win-x86;$(TargetRid) $(BuildArch) - false - false true + Release diff --git a/src/coreclr/tests/src/Directory.Build.targets b/src/coreclr/tests/src/Directory.Build.targets index cee75dcaf10e3b..920348b6cb0822 100644 --- a/src/coreclr/tests/src/Directory.Build.targets +++ b/src/coreclr/tests/src/Directory.Build.targets @@ -241,6 +241,7 @@ true $(BuildArch) + Release From 6041f29c8663cf9480283ccea72c0fae00506072 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 5 Dec 2019 16:37:13 -0800 Subject: [PATCH 43/71] Resolve ReferenceFromRuntime items directly from the CoreCLR output instead of via the runtime.depproj project. --- eng/pipelines/libraries/build-job.yml | 6 -- eng/referenceFromRuntime.targets | 58 ++++++------------- src/libraries/Directory.Build.props | 3 - src/libraries/Directory.Build.targets | 6 ++ src/libraries/restore/runtime/runtime.depproj | 2 +- 5 files changed, 25 insertions(+), 50 deletions(-) diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index bf020b9d7ee698..18718e0e8c05bc 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -111,12 +111,6 @@ jobs: sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/runtime targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin/runtime - - task: CopyFiles@2 - displayName: Prepare external.runtime folder to publish - inputs: - sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/external.runtime - targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin/external.runtime - - task: CopyFiles@2 displayName: Prepare shared framework ref assemblies to publish inputs: diff --git a/eng/referenceFromRuntime.targets b/eng/referenceFromRuntime.targets index a5a6a42ebd92fc..beaf200dce224a 100644 --- a/eng/referenceFromRuntime.targets +++ b/eng/referenceFromRuntime.targets @@ -1,53 +1,31 @@ - - - - - - - - - AddRuntimeProjectReference; - $(PrepareProjectReferencesDependsOn); - - - AddRuntimeProjectReference; - $(ResolveReferencesDependsOn); - - - AddRuntimeProjectReference; - $(CleanDependsOn) - - + - - - - false - _referencePathFromRuntime - - - - - - + <_referencePathFromRuntime Include="@(CoreCLRFiles)" /> + + <_referencePathFromRuntime Include="@(ReferenceFromRuntime->'$(RuntimePath)%(Identity).dll')" Condition="'$(IsTestProject)' == 'true'" /> <_referencePathFromRuntimeByFileName Include="@(_referencePathFromRuntime->'%(FileName)')" Condition="'%(_referencePathFromRuntime.Extension)' == '.dll'" > %(Identity) + + + + + + + - <_filteredReferencePathFromRuntimeByFileName Include="@(_referencePathFromRuntimeByFileName)" - Condition="'@(_referencePathFromRuntimeByFileName)' == '@(ReferenceFromRuntime)' and '%(Identity)' != ''"> + <_filteredReferencePathFromRuntimeByFileName Include="@(_referencePathFromRuntimeByFileNameFiltered)" + Condition="'@(_referencePathFromRuntimeByFileNameFiltered)' == '@(ReferenceFromRuntime)' and '%(Identity)' != ''"> @(ReferenceFromRuntime->'%(Aliases)') @@ -58,8 +36,8 @@ %(Identity) - <_filteredReferencePathFromRuntimeByFileName Include="@(_referencePathFromRuntimeByFileName)" - Condition="'@(_referencePathFromRuntimeByFileName)' == '@(_remainingReferenceFromRuntimeWithNI)' and '%(Identity)' != ''"> + <_filteredReferencePathFromRuntimeByFileName Include="@(_referencePathFromRuntimeByFileNameFiltered)" + Condition="'@(_referencePathFromRuntimeByFileNameFiltered)' == '@(_remainingReferenceFromRuntimeWithNI)' and '%(Identity)' != ''"> @(_remainingReferenceFromRuntimeWithNI->'%(Aliases)') diff --git a/src/libraries/Directory.Build.props b/src/libraries/Directory.Build.props index 83bc9633f2fd30..14b00bc14afcbc 100644 --- a/src/libraries/Directory.Build.props +++ b/src/libraries/Directory.Build.props @@ -267,9 +267,6 @@ $(TestArchiveTestsRoot)$(OSPlatformConfig)/ $(TestArchiveRoot)runtime/ - - $(MSBuildThisFileDirectory)restore\runtime\runtime.depproj - $(ArtifactsBinDir)pkg\netcoreapp\ref $(ArtifactsBinDir)pkg\netcoreapp\lib diff --git a/src/libraries/Directory.Build.targets b/src/libraries/Directory.Build.targets index 91ba959e6ba7cb..62e86b3ba37f9d 100644 --- a/src/libraries/Directory.Build.targets +++ b/src/libraries/Directory.Build.targets @@ -1,4 +1,10 @@ + + true + $(_bc_OSGroup) + $(_bc_ConfigurationGroup) + + diff --git a/src/libraries/restore/runtime/runtime.depproj b/src/libraries/restore/runtime/runtime.depproj index 8055e8010d4da2..f913baf7392128 100644 --- a/src/libraries/restore/runtime/runtime.depproj +++ b/src/libraries/restore/runtime/runtime.depproj @@ -7,7 +7,7 @@ true true $(DefaultOSGroup) - $(ConfigurationGroup) + $(ConfigurationGroup) false From a8e5503122edf950293896f6575e0b55336969f5 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 6 Dec 2019 12:51:32 -0800 Subject: [PATCH 44/71] Fix test host generation. Fix running the XML Serialization generator as part of libraries test build. --- eng/referenceFromRuntime.targets | 4 ++-- src/libraries/src.builds | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/eng/referenceFromRuntime.targets b/eng/referenceFromRuntime.targets index beaf200dce224a..eb560b54ba80ad 100644 --- a/eng/referenceFromRuntime.targets +++ b/eng/referenceFromRuntime.targets @@ -45,8 +45,8 @@ - - + + + + From b2f55dbc1f4ab7052c068f7fd6be42ca37c01803 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 6 Dec 2019 15:25:46 -0800 Subject: [PATCH 45/71] Fix XmlSerializer.Generator.Tests again. --- eng/referenceFromRuntime.targets | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/referenceFromRuntime.targets b/eng/referenceFromRuntime.targets index eb560b54ba80ad..e0dfc0a3500691 100644 --- a/eng/referenceFromRuntime.targets +++ b/eng/referenceFromRuntime.targets @@ -8,7 +8,7 @@ - <_referencePathFromRuntime Include="@(CoreCLRFiles)" /> + <_referencePathFromRuntime Include="@(CoreCLRFiles)" Private="false" /> <_referencePathFromRuntime Include="@(ReferenceFromRuntime->'$(RuntimePath)%(Identity).dll')" Condition="'$(IsTestProject)' == 'true'" /> @@ -45,8 +45,8 @@ - - + + Date: Fri, 6 Dec 2019 15:30:27 -0800 Subject: [PATCH 46/71] Don't look for CoreCLR files when using ReferenceFromRuntime and not using netcoreapp. --- eng/referenceFromRuntime.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/referenceFromRuntime.targets b/eng/referenceFromRuntime.targets index e0dfc0a3500691..5891df84636b90 100644 --- a/eng/referenceFromRuntime.targets +++ b/eng/referenceFromRuntime.targets @@ -8,7 +8,7 @@ - <_referencePathFromRuntime Include="@(CoreCLRFiles)" Private="false" /> + <_referencePathFromRuntime Include="@(CoreCLRFiles)" Private="false" Condition="$(TargetFramework.StartsWith('netcoreapp'))" /> <_referencePathFromRuntime Include="@(ReferenceFromRuntime->'$(RuntimePath)%(Identity).dll')" Condition="'$(IsTestProject)' == 'true'" /> From 3012c846234ef56e3a34b99d760d7caf6daaddae Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 6 Dec 2019 15:34:36 -0800 Subject: [PATCH 47/71] Crossgen-comparison should depend on a libraries build so we can populate Core_Root. --- eng/pipelines/coreclr/pr.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/pipelines/coreclr/pr.yml b/eng/pipelines/coreclr/pr.yml index f31456d84a43d7..0d8691ff7eb0e9 100644 --- a/eng/pipelines/coreclr/pr.yml +++ b/eng/pipelines/coreclr/pr.yml @@ -174,6 +174,8 @@ jobs: - Linux_arm helixQueueGroup: pr helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + liveLibrariesBuildConfig: Release # # Release test builds From 6f666683723d39a4e8ee4fe40a6671ff192b0d72 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 6 Dec 2019 15:48:43 -0800 Subject: [PATCH 48/71] Add job dependency on libraries build from run-test-job to ensure we wait correctly. Otherwise we might have a race since the managed test build might depend on one libraries build while the test run depends on another. --- eng/pipelines/coreclr/templates/run-test-job.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/templates/run-test-job.yml b/eng/pipelines/coreclr/templates/run-test-job.yml index c89d7a0d3590b8..a5db125e48e02c 100644 --- a/eng/pipelines/coreclr/templates/run-test-job.yml +++ b/eng/pipelines/coreclr/templates/run-test-job.yml @@ -56,6 +56,8 @@ jobs: - 'coreclr_test_build_p1_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{parameters.buildConfig }}' - ${{ if ne(parameters.stagedBuild, true) }}: - ${{ format('coreclr_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: + - ${{ format('libraries_build_{0}_{1}{2}_{3}_{4}', 'netcoreapp', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} # Compute job name from template parameters ${{ if eq(parameters.testGroup, 'innerloop') }}: @@ -91,7 +93,7 @@ jobs: - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - group: DotNet-HelixApi-Access - + - ${{ parameters.variables }} # TODO: update these numbers as they were determined long ago @@ -383,4 +385,4 @@ jobs: pathtoPublish: $(Build.SourcesDirectory)/artifacts/log artifactName: '$(LogNamePrefix)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_${{ parameters.testGroup }}' continueOnError: true - condition: always() \ No newline at end of file + condition: always() From a19f8980f36896d640df53cebf9d0732bfdf873a Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 6 Dec 2019 16:52:47 -0800 Subject: [PATCH 49/71] Only resolve coreclr files from local build when targetting netcoreapp. --- eng/referenceFromRuntime.targets | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/eng/referenceFromRuntime.targets b/eng/referenceFromRuntime.targets index 5891df84636b90..11d012b9f00a70 100644 --- a/eng/referenceFromRuntime.targets +++ b/eng/referenceFromRuntime.targets @@ -1,14 +1,19 @@ + + + + + + - <_referencePathFromRuntime Include="@(CoreCLRFiles)" Private="false" Condition="$(TargetFramework.StartsWith('netcoreapp'))" /> + <_referencePathFromRuntime Include="@(CoreCLRFiles)" Private="false" /> <_referencePathFromRuntime Include="@(ReferenceFromRuntime->'$(RuntimePath)%(Identity).dll')" Condition="'$(IsTestProject)' == 'true'" /> From 4d1af1e1e761414468303c602398512700c92811 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 6 Dec 2019 17:42:54 -0800 Subject: [PATCH 50/71] Add steps to download libraries build for crossgen-comparison. --- .../templates/crossgen-comparison-job.yml | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml index b90d31ada69674..4c032e6c19de5d 100644 --- a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml @@ -10,6 +10,13 @@ parameters: variables: {} pool: '' + # When set to a non-empty value (Debug / Release), it determines libraries + # build configuration to use for the tests. Setting this property implies + # a dependency of this job on the appropriate libraries build and is used + # to construct the name of the Azure artifact representing libraries build + # to use for building the tests. + liveLibrariesBuildConfig: '' + ### Crossgen-comparison job ### ### Ensure that the output of cross-architecture, e.g. x64-hosted-arm-targeting, @@ -23,6 +30,7 @@ jobs: osGroup: ${{ parameters.osGroup }} osSubgroup: ${{ parameters.osSubgroup }} stagedBuild: ${{ parameters.stagedBuild }} + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} helixType: 'test/crossgen-comparison/' pool: ${{ parameters.pool }} @@ -59,7 +67,10 @@ jobs: - ${{ parameters.variables }} # Test job depends on the corresponding build job - dependsOn: ${{ format('coreclr_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + dependsOn: + - ${{ format('coreclr_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: + - ${{ format('libraries_build_{0}_{1}{2}_{3}_{4}', 'netcoreapp', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} # Run all steps in the container. # Note that the containers are defined in platform-matrix.yml @@ -78,6 +89,16 @@ jobs: artifactName: '$(buildProductArtifactName)' displayName: 'product build' + # Optionally download live-built libraries + - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(librariesDownloadDir) + cleanUnpackFolder: false + artifactFileName: '$(librariesBuildArtifactName)$(archiveExtension)' + artifactName: '$(librariesBuildArtifactName)' + displayName: 'live-built libraries' + # Create directories and ensure crossgen is executable - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: From 7b46f81169416a7f7fd2bd3aea83fb6b72e3e9ac Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 6 Dec 2019 17:55:00 -0800 Subject: [PATCH 51/71] Revive old method of resolving references to the runtime for building against old netcoreapp (ie netcoreapp 3.0). Fix correctly passing along aliases info. --- eng/referenceFromRuntime.targets | 39 +++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/eng/referenceFromRuntime.targets b/eng/referenceFromRuntime.targets index 11d012b9f00a70..33af01af354c4d 100644 --- a/eng/referenceFromRuntime.targets +++ b/eng/referenceFromRuntime.targets @@ -1,19 +1,51 @@ + + + AddRuntimeProjectReference; + $(PrepareProjectReferencesDependsOn); + + + AddRuntimeProjectReference; + $(ResolveReferencesDependsOn); + + + AddRuntimeProjectReference; + $(CleanDependsOn) + + + + + + + + $([MSBuild]::NormalizePath('$(LibrariesProjectRoot)', 'restore', 'runtime', 'runtime.depproj')) + + + + + false + _referencePathFromRestoredRuntime + + + + - + <_referencePathFromRuntime Include="@(CoreCLRFiles)" Private="false" /> + <_referencePathFromRuntime Include="@(_referencePathFromRestoredRuntime)" Private="false" /> <_referencePathFromRuntime Include="@(ReferenceFromRuntime->'$(RuntimePath)%(Identity).dll')" Condition="'$(IsTestProject)' == 'true'" /> @@ -50,8 +82,9 @@ + - + Date: Mon, 9 Dec 2019 11:19:28 -0800 Subject: [PATCH 52/71] Fix failing AllConfigurations leg that was due to the netcoreapp runtime.depproj getting binplaced in the runtime output folder. --- src/libraries/Directory.Build.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/Directory.Build.targets b/src/libraries/Directory.Build.targets index 62e86b3ba37f9d..0b38649cba71da 100644 --- a/src/libraries/Directory.Build.targets +++ b/src/libraries/Directory.Build.targets @@ -89,7 +89,7 @@ $(RefRootPath)%(Identity)/ - + $(ArtifactsBinDir)runtime/%(Identity)-$(ConfigurationGroup)-$(ArchGroup) From 613e369215d74a016878317f82ad36ddfa09ae21 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 9 Dec 2019 12:26:07 -0800 Subject: [PATCH 53/71] Update eng/liveBuilds.targets Co-Authored-By: Davis Goodin --- eng/liveBuilds.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets index 938c72502e1035..abe13d463da81f 100644 --- a/eng/liveBuilds.targets +++ b/eng/liveBuilds.targets @@ -24,7 +24,7 @@ $([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)')) $([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)','sharedFramework')) $([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)','$(CoreCLRCrossTargetComponentDir)','sharedFramework')) + Condition="'$(CoreCLRCrossTargetComponentDirName)' != ''">$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)','$(CoreCLRCrossTargetComponentDirName)','sharedFramework')) From 27e34046988b262c5e32afe302f002f9b077ccaa Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 9 Dec 2019 12:43:00 -0800 Subject: [PATCH 54/71] HostArch->BuildArchitecture and share with installer. --- Directory.Build.props | 2 ++ eng/liveBuilds.targets | 7 +++---- src/installer/Directory.Build.props | 1 - 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 23b14893a0d990..6c65af67d082d6 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -90,6 +90,8 @@ true Properties + + $([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant()) diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets index 938c72502e1035..9b013eab292d8b 100644 --- a/eng/liveBuilds.targets +++ b/eng/liveBuilds.targets @@ -1,7 +1,6 @@ - $([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant) $(OSGroup) $(Configuration) $(OSGroup) @@ -12,9 +11,9 @@ $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'pkg', 'netcoreapp', 'runtime', '$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'runtime', 'netcoreapp-$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'native', 'netcoreapp-$(LibrariesOSGroup)-$(LibrariesConfiguration)-$(TargetArchitecture)')) - x64 - x86 - x64 + x64 + x86 + x64 diff --git a/src/installer/Directory.Build.props b/src/installer/Directory.Build.props index 9e02e54e5cc7c4..4ee971aa8d2485 100644 --- a/src/installer/Directory.Build.props +++ b/src/installer/Directory.Build.props @@ -45,7 +45,6 @@ - $([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant()) $(BuildArchitecture) x64 $(TargetArchitecture) From 8bbdc41e3773183b9dce22921e15042ef9ce9a24 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 9 Dec 2019 14:34:24 -0800 Subject: [PATCH 55/71] Fix format nits. --- src/libraries/Directory.Build.targets | 4 ++-- src/libraries/restore/runtime/runtime.depproj | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libraries/Directory.Build.targets b/src/libraries/Directory.Build.targets index 0b38649cba71da..c789596f3d6993 100644 --- a/src/libraries/Directory.Build.targets +++ b/src/libraries/Directory.Build.targets @@ -1,8 +1,8 @@ true - $(_bc_OSGroup) - $(_bc_ConfigurationGroup) + $(_bc_OSGroup) + $(_bc_ConfigurationGroup) diff --git a/src/libraries/restore/runtime/runtime.depproj b/src/libraries/restore/runtime/runtime.depproj index f913baf7392128..776f22ca840c85 100644 --- a/src/libraries/restore/runtime/runtime.depproj +++ b/src/libraries/restore/runtime/runtime.depproj @@ -6,8 +6,8 @@ $(NoWarn);NU1603;NU1605 true true - $(DefaultOSGroup) - $(ConfigurationGroup) + $(DefaultOSGroup) + $(ConfigurationGroup) false From a63b863960771941b5c109a37170e51d3a15de39 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 9 Dec 2019 14:35:24 -0800 Subject: [PATCH 56/71] Update YAML variable name. --- eng/pipelines/libraries/base-job.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index 698214f1c27714..437a496e8724a8 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -69,12 +69,12 @@ jobs: - _coreClrArtifactName: '' - _coreClrDownloadPath: '' - - _coreClrOverridePathArg: '' + - _coreClrArtifactsPathArg: '' - ${{ if ne(parameters.liveCoreClrBuildConfig, '') }}: - _coreClrDownloadPath: '$(Build.SourcesDirectory)/artifacts/transport/coreclr' - _coreClrArtifactName: 'CoreCLRProduct_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveCoreClrBuildConfig }}' - - _coreClrOverridePathArg: ' /p:CoreCLRArtifactsPath=$(_coreClrDownloadPath)' + - _coreClrArtifactsPathArg: ' /p:CoreCLRArtifactsPath=$(_coreClrDownloadPath)' # WebAssembly uses linux implementation detail - ${{ if eq(parameters.osGroup, 'WebAssembly') }}: @@ -90,7 +90,7 @@ jobs: - ${{ if eq(parameters.isOfficialBuild, 'true') }}: - _stripSymbolsArg: -stripSymbols - - _buildArguments: -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) $(_stripSymbolsArg) $(_testScopeArg) $(_warnAsErrorArg) $(_runtimeOSArg) $(_msbuildCommonParameters) $(_coreClrOverridePathArg) + - _buildArguments: -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) $(_stripSymbolsArg) $(_testScopeArg) $(_warnAsErrorArg) $(_runtimeOSArg) $(_msbuildCommonParameters) $(_coreClrArtifactsPathArg) - ${{ parameters.variables }} dependsOn: From 7788dbef9a8572face75b96f232416b2e3e1f7f3 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 9 Dec 2019 14:38:19 -0800 Subject: [PATCH 57/71] Remove setting property to import liveBuilds.targets. --- Directory.Build.targets | 2 +- src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj | 1 - src/libraries/Directory.Build.targets | 1 - src/libraries/restore/runtime/runtime.depproj | 1 - src/libraries/shims/manual/Directory.Build.props | 1 - 5 files changed, 1 insertion(+), 5 deletions(-) diff --git a/Directory.Build.targets b/Directory.Build.targets index d232c6fd9bb511..a1589f9f5d5661 100644 --- a/Directory.Build.targets +++ b/Directory.Build.targets @@ -8,7 +8,7 @@ false - + diff --git a/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj b/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj index ced5fb1500f081..bdc57253c648a5 100644 --- a/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj +++ b/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj @@ -2,7 +2,6 @@ Microsoft.NETCore.App - true false Windows_NT Linux diff --git a/src/libraries/Directory.Build.targets b/src/libraries/Directory.Build.targets index c789596f3d6993..eab50eee6615e5 100644 --- a/src/libraries/Directory.Build.targets +++ b/src/libraries/Directory.Build.targets @@ -1,6 +1,5 @@ - true $(_bc_OSGroup) $(_bc_ConfigurationGroup) diff --git a/src/libraries/restore/runtime/runtime.depproj b/src/libraries/restore/runtime/runtime.depproj index 776f22ca840c85..d6520703f7972b 100644 --- a/src/libraries/restore/runtime/runtime.depproj +++ b/src/libraries/restore/runtime/runtime.depproj @@ -5,7 +5,6 @@ $(ToolRuntimeRID) $(NoWarn);NU1603;NU1605 true - true $(DefaultOSGroup) $(ConfigurationGroup) false diff --git a/src/libraries/shims/manual/Directory.Build.props b/src/libraries/shims/manual/Directory.Build.props index c6c1f72b25758f..d133e1342d10b7 100644 --- a/src/libraries/shims/manual/Directory.Build.props +++ b/src/libraries/shims/manual/Directory.Build.props @@ -13,7 +13,6 @@ true $(ArtifactsBinDir)runtime/$(TargetGroup)-$(_bc_OSGroup)-$(_bc_ConfigurationGroup)-$(ArchGroup)/ - true $(_bc_OSGroup) $(_bc_ConfigurationGroup) From b37d7548abd5258e985c1f650375456f1044c2b6 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 10 Dec 2019 18:12:04 -0800 Subject: [PATCH 58/71] Upload docs and packages folders from libraries build --- eng/pipelines/libraries/build-job.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index e40924f6852d4c..ddfa9676fceb84 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -117,11 +117,18 @@ jobs: sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/ref/microsoft.netcore.app targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin/ref/microsoft.netcore.app + - task: CopyFiles@2 displayName: Prepare shared framework runtime folder to publish inputs: sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/pkg/${{ parameters.framework }}/runtime targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin/pkg/${{ parameters.framework }}/runtime + - task: CopyFiles@2 + displayName: Prepare docs folder to publish + inputs: + sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/docs + targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin/docs + - task: CopyFiles@2 displayName: Prepare native folder to publish inputs: @@ -140,6 +147,12 @@ jobs: sourceFolder: $(Build.SourcesDirectory)/artifacts/toolset targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/toolset + - task: CopyFiles@2 + displayName: Prepare artifacts packages folder to publish + inputs: + sourceFolder: $(Build.SourcesDirectory)/artifacts/packages + targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/packages + - template: /eng/pipelines/common/upload-artifact-step.yml parameters: rootFolder: $(Build.ArtifactStagingDirectory)/artifacts From 3cbbb3ea9586dca934b1e40348b3656a89846546 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 10 Dec 2019 18:12:39 -0800 Subject: [PATCH 59/71] Inclue doc files in the Microsoft.NETCore.App.Internal package --- src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj b/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj index 2cc2e980757139..cba7a50cce39e9 100644 --- a/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj +++ b/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj @@ -52,6 +52,10 @@ + + + <_docFilesToPackage Include="$(ArtifactsBinDir)/docs/%(LibrariesRefAssemblies.FileName).xml" Condition="Exists('$(ArtifactsBinDir)/docs/%(LibrariesRefAssemblies.FileName).xml')"/> + From d30709548920e05602187cba27d7562acf3349c0 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 10 Dec 2019 18:26:13 -0800 Subject: [PATCH 60/71] Try making the runtime-installer pipeline live-live using the build-coreclr-and-libraries-job template. --- eng/pipelines/installer/azure-pipelines.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/eng/pipelines/installer/azure-pipelines.yml b/eng/pipelines/installer/azure-pipelines.yml index 8504d0acf463e1..0f8562c5e801d4 100644 --- a/eng/pipelines/installer/azure-pipelines.yml +++ b/eng/pipelines/installer/azure-pipelines.yml @@ -50,6 +50,15 @@ jobs: # - template: /eng/pipelines/common/checkout-job.yml +# +# Build CoreCLR and Libraries +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + buildConfig: release + platformGroup: all + # # Build and Test # @@ -61,3 +70,5 @@ jobs: _BuildConfig: Debug release: _BuildConfig: Release + liveCoreClrBuildConfig: release + liveLibrariesBuildConfig: Release From fbdde3e321d8356a9b1e62a4f020d6fc08d26c92 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 11 Dec 2019 10:41:04 -0800 Subject: [PATCH 61/71] Update artifacts download paths and msbuild properties for installer build. --- eng/pipelines/installer/jobs/base-job.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/installer/jobs/base-job.yml b/eng/pipelines/installer/jobs/base-job.yml index 033c518af0e6bd..06f283b8288346 100644 --- a/eng/pipelines/installer/jobs/base-job.yml +++ b/eng/pipelines/installer/jobs/base-job.yml @@ -56,12 +56,12 @@ jobs: - name: LiveOverridePathArgs value: >- - $(CoreCLROverridePathArg) - $(CoreFXOverridePathArg) + $(CoreCLRArtifactsPathArg) + $(LibrariesConfigurationArg) - - name: CoreCLROverridePathArg + - name: CoreCLRArtifactsPathArg value: '' - - name: CoreFXOverridePathArg + - name: LibrariesConfigurationArg value: '' - name: CoreClrDownloadPath @@ -78,8 +78,8 @@ jobs: parameters.liveCoreClrBuildConfig) }} - name: CoreClrDownloadPath value: 'artifacts/transport/coreclr' - - name: CoreCLROverridePathArg - value: /p:CoreCLROverridePath=${{ parameters.buildCommandSourcesDirectory }}$(CoreClrDownloadPath) + - name: CoreCLRArtifactsPathArg + value: /p:CoreCLRArtifactsPath=${{ parameters.buildCommandSourcesDirectory }}$(CoreClrDownloadPath) - name: CoreClrArtifactName value: CoreCLRProduct_$(liveCoreClrLegName) @@ -91,11 +91,11 @@ jobs: parameters.archType, parameters.liveLibrariesBuildConfig) }} - name: LibrariesDownloadPath - value: 'artifacts/transport/libraries' - - name: CoreFXOverridePathArg - value: /p:CoreFXOverridePath=${{ parameters.buildCommandSourcesDirectory }}$(LibrariesDownloadPath) + value: 'artifacts' - name: LibrariesArtifactName value: libraries_bin_$(liveLibrariesLegName) + - name: LibrariesConfigurationArg + value: ' /p:LibrariesConfiguration=${{ parameters.liveLibrariesBuildConfig }}' dependsOn: - checkout From 60ecad151ba525606111bdafb1f7e6cb3fd89786 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 11 Dec 2019 11:08:23 -0800 Subject: [PATCH 62/71] Mark cross-component files as native. --- eng/liveBuilds.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets index 76ff46e1005ac4..574d9bedead645 100644 --- a/eng/liveBuilds.targets +++ b/eng/liveBuilds.targets @@ -27,7 +27,7 @@ - + true From 9ae4d085751dfe5621057101da4cde866f21f692 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 11 Dec 2019 11:09:08 -0800 Subject: [PATCH 63/71] Remove FrameworkPackageName to remove error about missing $(FrameworkPackageName).versions.txt file that other shared frameworks also don't generate. --- src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj b/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj index cba7a50cce39e9..f3f34b34ce449f 100644 --- a/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj +++ b/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj @@ -1,7 +1,6 @@ - Microsoft.NETCore.App false Windows_NT Linux From cd636c83a435e4b57f77a088fd990c679d0c688b Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 11 Dec 2019 11:10:12 -0800 Subject: [PATCH 64/71] Restore GetFilesFromPackageResolve override. --- .../src/localnetcoreapp.override.targets | 48 ++++++++++++++++++- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/src/installer/pkg/projects/netcoreapp/src/localnetcoreapp.override.targets b/src/installer/pkg/projects/netcoreapp/src/localnetcoreapp.override.targets index bd57e83538c682..52f64d3386d390 100644 --- a/src/installer/pkg/projects/netcoreapp/src/localnetcoreapp.override.targets +++ b/src/installer/pkg/projects/netcoreapp/src/localnetcoreapp.override.targets @@ -39,7 +39,6 @@ <_diaSymReaderAssembly Include="$(_diaSymReaderPackageDir)**\Microsoft.DiaSymReader.Native.*.dll" /> - <_runtimeDirectory>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', 'sharedFramework')) <_crossGenPath>$([MSBuild]::NormalizePath('$(CoreCLRArtifactsPath)', '$(CoreCLRCrossTargetComponentDirName)', 'sharedFramework', 'crossgen$(ApplicationFileExtension)')) @@ -82,7 +81,6 @@ <_crossgenPlatformDirectories Include="%(_filesToCrossGen.RootDir)%(_filesToCrossGen.Directory)" /> <_crossgenPlatformDirectories Include="$(_runtimeDirectory)" /> - <_crossgenPlatformDirectories Include="$(_coreLibDirectory)" /> <_crossgenPlatformDirectories Include="$(_fxLibDirectory)" /> @@ -94,4 +92,50 @@ <_crossgenPlatformAssemblies>@(_crossgenPlatformDirectories->'%(Identity)', '$(_pathSeparatorEscaped)') + + + + + + + true + + + + + runtimes/$(PackageRID)/lib/$(PackageTargetFramework) + runtimes/$(PackageRID)/native + + + + %(RidSpecificFilesToPackage.TargetPath)/%(RidSpecificFilesToPackage.DestinationSubDirectory) + + + + + + + + + + + ref/$(PackageTargetFramework) + + + + <_docFilesToPackage Include="%(FilesToPackage.RootDir)%(FilesToPackage.Directory)**\%(FilesToPackage.FileName).xml" /> + + + ref/$(PackageTargetFramework)/%(RecursiveDir) + + + + + + + + + + From 77f3ec72a6044f63292385b201fad023af0b1822 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 11 Dec 2019 14:27:13 -0800 Subject: [PATCH 65/71] Don't clean the unpack folder for the libraries download. It wipes out the CoreCLR download. --- eng/pipelines/installer/jobs/base-job.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/installer/jobs/base-job.yml b/eng/pipelines/installer/jobs/base-job.yml index 06f283b8288346..d72d0bccd98f5d 100644 --- a/eng/pipelines/installer/jobs/base-job.yml +++ b/eng/pipelines/installer/jobs/base-job.yml @@ -152,6 +152,7 @@ jobs: artifactFileName: '$(LibrariesArtifactName)$(archiveExtension)' artifactName: '$(LibrariesArtifactName)' displayName: 'Libraries artifacts' + cleanUnpackFolder: false - ${{ parameters.buildSteps }} From 7257b1ef942504b2283e1e269475b48bc92bc109 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 11 Dec 2019 16:02:16 -0800 Subject: [PATCH 66/71] Make sure that we restore the WinRT targetting pack during the installer build. --- .../pkg/projects/Directory.Build.targets | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/installer/pkg/projects/Directory.Build.targets b/src/installer/pkg/projects/Directory.Build.targets index 46319d4d1e34a6..f8ca05a3026db9 100644 --- a/src/installer/pkg/projects/Directory.Build.targets +++ b/src/installer/pkg/projects/Directory.Build.targets @@ -336,6 +336,26 @@ - - + + + + + + + + + + + + From cdf9a291850ce1299b88f89e84eb5222165b9632 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 11 Dec 2019 17:05:42 -0800 Subject: [PATCH 67/71] Add libraries package directory to RestoreSources for the case where the LibrariesConfiguration is different from the installer build configuration. --- .../sfx/Microsoft.NETCore.App.SharedFx.sfxproj | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/installer/pkg/projects/netcoreapp/sfx/Microsoft.NETCore.App.SharedFx.sfxproj b/src/installer/pkg/projects/netcoreapp/sfx/Microsoft.NETCore.App.SharedFx.sfxproj index c24c1a8db2e472..79b2e4b24ebd36 100644 --- a/src/installer/pkg/projects/netcoreapp/sfx/Microsoft.NETCore.App.SharedFx.sfxproj +++ b/src/installer/pkg/projects/netcoreapp/sfx/Microsoft.NETCore.App.SharedFx.sfxproj @@ -1,4 +1,5 @@ - + + Microsoft.NETCore.App @@ -42,4 +43,18 @@ + + + + + $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'packages', '$(LibrariesConfiguration)')) + $([MSBuild]::NormalizeDirectory('$(LibrariesPackagesDir)', 'Shipping')) + $([MSBuild]::NormalizeDirectory('$(LibrariesPackagesDir)', 'NonShipping')) + + $(RestoreSources);$(LibrariesShippingPackagesDir) + $(RestoreSources);$(LibrariesNonShippingPackagesDir) + From 9b62aff2ca4c333a8750bfb2238911789184289a Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 11 Dec 2019 17:22:57 -0800 Subject: [PATCH 68/71] Don't pass OfficialBuildId unless it is an official build. We want to be consistent with the libraries build so our package versions match. --- eng/pipelines/installer/jobs/base-job.yml | 2 ++ eng/pipelines/installer/jobs/bash-build.yml | 2 +- eng/pipelines/installer/jobs/osx-build.yml | 2 +- eng/pipelines/installer/jobs/windows-build.yml | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/installer/jobs/base-job.yml b/eng/pipelines/installer/jobs/base-job.yml index d72d0bccd98f5d..c2905ab9ff7424 100644 --- a/eng/pipelines/installer/jobs/base-job.yml +++ b/eng/pipelines/installer/jobs/base-job.yml @@ -53,6 +53,8 @@ jobs: - ${{ if eq(variables.isOfficialBuild, true) }}: - name: SignType value: $[ coalesce(variables.OfficialSignType, 'real') ] + - name: OfficialBuildArg + value: /p:OfficialBuildId=$(Build.BuildNumber) - name: LiveOverridePathArgs value: >- diff --git a/eng/pipelines/installer/jobs/bash-build.yml b/eng/pipelines/installer/jobs/bash-build.yml index 0812658c97b1ab..c30ed40b2b295f 100644 --- a/eng/pipelines/installer/jobs/bash-build.yml +++ b/eng/pipelines/installer/jobs/bash-build.yml @@ -55,8 +55,8 @@ jobs: CommonMSBuildArgs: >- /p:Configuration=$(_BuildConfig) - /p:OfficialBuildId=$(Build.BuildNumber) /p:TargetArchitecture=${{ parameters.archType }} + $(OfficialBuildArg) # Don't put additionalMSBuildArgs as the last line. It may or may not have extra args. If the # parameter is empty, AzDO replaces it with empty space without chomping the extra newline. diff --git a/eng/pipelines/installer/jobs/osx-build.yml b/eng/pipelines/installer/jobs/osx-build.yml index 6c663d36cf5fa4..bff6c67b213828 100644 --- a/eng/pipelines/installer/jobs/osx-build.yml +++ b/eng/pipelines/installer/jobs/osx-build.yml @@ -17,9 +17,9 @@ jobs: - script: >- $(Build.SourcesDirectory)/installer.sh --restore --build --ci --test - /p:OfficialBuildId=$(Build.BuildNumber) /p:StripSymbols=true $(LiveOverridePathArgs) $(CommonMSBuildArgs) + $(OfficialBuildArg) displayName: Build condition: succeeded() diff --git a/eng/pipelines/installer/jobs/windows-build.yml b/eng/pipelines/installer/jobs/windows-build.yml index 82f1a6310e4014..9126eed56cd8c1 100644 --- a/eng/pipelines/installer/jobs/windows-build.yml +++ b/eng/pipelines/installer/jobs/windows-build.yml @@ -15,10 +15,10 @@ jobs: buildVariables: CommonMSBuildArgs: >- /p:Configuration=$(_BuildConfig) - /p:OfficialBuildId=$(Build.BuildNumber) /p:TargetArchitecture=${{ parameters.archType }} /p:PortableBuild=true /p:SkipTests=${{ parameters.skipTests }} + $(OfficialBuildArg) MsbuildSigningArguments: >- /p:CertificateId=400 /p:DotNetSignType=$(SignType) From 35bfb81dadf6f7d459ae2d8295066bc63bace80f Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 11 Dec 2019 17:43:12 -0800 Subject: [PATCH 69/71] Give default value for OfficialBuildArg variable --- eng/pipelines/installer/jobs/base-job.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eng/pipelines/installer/jobs/base-job.yml b/eng/pipelines/installer/jobs/base-job.yml index c2905ab9ff7424..34ba20e8e4c3ff 100644 --- a/eng/pipelines/installer/jobs/base-job.yml +++ b/eng/pipelines/installer/jobs/base-job.yml @@ -44,6 +44,9 @@ jobs: - name: ${{ variable.key }} value: ${{ variable.value }} + - name: OfficialBuildArg + value: '' + # Produce test-signed build for PR and Public builds - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - name: SignType From ef5144cad8dd7ef175120fb326c9ee3dcabab5b3 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 12 Dec 2019 13:54:00 -0800 Subject: [PATCH 70/71] Ensure installer tests can see the libraries-generated packages. Fix typo in TargetPath for CoreCLR assets (runtime instead of runtimes). Fix paths for CoreCLR cross-target files as well as crossgen to place them in the correct directories. --- eng/liveBuilds.targets | 4 ---- src/installer/Directory.Build.targets | 13 ++++++++++--- .../Microsoft.NETCore.App.SharedFx.sfxproj | 4 ---- .../netcoreapp/src/netcoreapp.depproj | 19 ++++++++++++++++++- .../PrepareTestAssets/PrepareTestAssets.proj | 2 ++ 5 files changed, 30 insertions(+), 12 deletions(-) diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets index 574d9bedead645..63d8d1b1a20490 100644 --- a/eng/liveBuilds.targets +++ b/eng/liveBuilds.targets @@ -47,10 +47,6 @@ $(CoreCLRCrossTargetComponentDir)PDB/*.pdb; $(CoreCLRCrossTargetComponentDir)PDB/*.dbg; $(CoreCLRCrossTargetComponentDir)PDB/*.dwarf" /> - - - runtime/$(CoreCLRCrossTargetComponentDirName)_$(TargetArchitecture)/native - diff --git a/src/installer/Directory.Build.targets b/src/installer/Directory.Build.targets index 724a3f5557e2c1..3d45d9ee49c92b 100644 --- a/src/installer/Directory.Build.targets +++ b/src/installer/Directory.Build.targets @@ -11,6 +11,13 @@ + + + $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'packages', '$(LibrariesConfiguration)')) + $([MSBuild]::NormalizeDirectory('$(LibrariesPackagesDir)', 'Shipping')) + $([MSBuild]::NormalizeDirectory('$(LibrariesPackagesDir)', 'NonShipping')) + + @@ -128,12 +135,12 @@ $(ProductionVersion) - + 0.1.$(VersionSuffix) $([System.String]::Copy('$(HostPackageRelease)').Replace('-', '_')) - + $(ProductionVersion) - + 0.1.$(VersionSuffix) $([System.String]::Copy('$(HostResolverPackageRelease)').Replace('-', '_')) diff --git a/src/installer/pkg/projects/netcoreapp/sfx/Microsoft.NETCore.App.SharedFx.sfxproj b/src/installer/pkg/projects/netcoreapp/sfx/Microsoft.NETCore.App.SharedFx.sfxproj index 79b2e4b24ebd36..3ea98ed5d6c64f 100644 --- a/src/installer/pkg/projects/netcoreapp/sfx/Microsoft.NETCore.App.SharedFx.sfxproj +++ b/src/installer/pkg/projects/netcoreapp/sfx/Microsoft.NETCore.App.SharedFx.sfxproj @@ -50,10 +50,6 @@ to be able to find packages produced in the libraries phase and consumed in the installer phase. --> - $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'packages', '$(LibrariesConfiguration)')) - $([MSBuild]::NormalizeDirectory('$(LibrariesPackagesDir)', 'Shipping')) - $([MSBuild]::NormalizeDirectory('$(LibrariesPackagesDir)', 'NonShipping')) - $(RestoreSources);$(LibrariesShippingPackagesDir) $(RestoreSources);$(LibrariesNonShippingPackagesDir) diff --git a/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj b/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj index f3f34b34ce449f..802daf0d9e280b 100644 --- a/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj +++ b/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj @@ -34,8 +34,25 @@ - runtime/$(PackageRID)/native + runtimes/$(PackageRID)/native + + + tools + + + + runtimes/$(CoreCLRCrossTargetComponentDirName)_$(TargetArchitecture)/native + + + tools/$(CoreCLRCrossTargetComponentDirName)_$(TargetArchitecture) + + + tools/$(CoreCLRCrossTargetComponentDirName)_$(TargetArchitecture) + + + tools/$(CoreCLRCrossTargetComponentDirName)_$(TargetArchitecture) + diff --git a/src/installer/test/PrepareTestAssets/PrepareTestAssets.proj b/src/installer/test/PrepareTestAssets/PrepareTestAssets.proj index b2ac07d4cebfc1..46b4ba47107341 100644 --- a/src/installer/test/PrepareTestAssets/PrepareTestAssets.proj +++ b/src/installer/test/PrepareTestAssets/PrepareTestAssets.proj @@ -49,6 +49,8 @@ + + Date: Thu, 12 Dec 2019 15:38:46 -0800 Subject: [PATCH 71/71] Add back FrameworkPackageName to get the partial platform manifest to generate. --- src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj b/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj index 802daf0d9e280b..b5edf4df303b12 100644 --- a/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj +++ b/src/installer/pkg/projects/netcoreapp/src/netcoreapp.depproj @@ -1,6 +1,7 @@ + Microsoft.NETCore.App false Windows_NT Linux