diff --git a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/CreateTrimDependencyGroups.cs b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/CreateTrimDependencyGroups.cs index 5452ba9a17..d5d11e56bb 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/CreateTrimDependencyGroups.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/CreateTrimDependencyGroups.cs @@ -15,26 +15,30 @@ namespace Microsoft.DotNet.Build.Tasks.Packaging public class CreateTrimDependencyGroups : PackagingTask { [Required] - public string FrameworkListsPath + public ITaskItem[] Dependencies { get; set; } [Required] - public ITaskItem[] Dependencies + public ITaskItem[] Files { get; set; } + /// + /// Package index files used to define stable package list. + /// [Required] - public ITaskItem[] Files + public ITaskItem[] PackageIndexes { get; set; } + [Output] public ITaskItem[] TrimmedDependencies { @@ -53,12 +57,14 @@ public override bool Execute() Log.LogError("Dependencies argument must be specified"); return false; } - if (null == FrameworkListsPath) + if (PackageIndexes == null && PackageIndexes.Length == 0) { - Log.LogError("FrameworkListsPath argument must be specified"); + Log.LogError("PackageIndexes argument must be specified"); return false; } + var index = PackageIndex.Load(PackageIndexes.Select(pi => pi.GetMetadata("FullPath"))); + // Retrieve the list of generation dependency group TFM's var dependencyGroups = Dependencies.GroupBy(d => d.GetMetadata("TargetFramework")).Select(dg => new { @@ -82,7 +88,7 @@ public override bool Execute() { // Determine inbox frameworks for this generation that don't already have explicit groups HashSet inboxFrameworksList = new HashSet( - Frameworks.GetAlllInboxFrameworks(FrameworkListsPath) + index.GetAlllInboxFrameworks() .Where(fx => !fx.IsPCL) .Where(fx => Generations.DetermineGenerationForFramework(fx, UseNetPlatform) >= portableDependencyGroup.Framework.Version && !frameworksToExclude.Any(exFx => exFx.Framework == fx.Framework && exFx.Version <= fx.Version))); @@ -103,7 +109,7 @@ public override bool Execute() { string version = GetVersion(dependency); - if (!Frameworks.IsInbox(FrameworkListsPath, framework, dependency.ItemSpec, version)) + if (!index.IsInbox(dependency.ItemSpec, framework, version)) { addedDependencyToFramework = true; AddDependency(addedDependencies, new TaskItem(dependency), framework, portableDependencyGroup.Framework); diff --git a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/GeneratePackageReport.cs b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/GeneratePackageReport.cs index e8409235a3..aaa412796e 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/GeneratePackageReport.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/GeneratePackageReport.cs @@ -60,7 +60,7 @@ public ITaskItem[] Frameworks public string RuntimeFile { get; set; } [Required] - public string FrameworkListsPath + public ITaskItem[] PackageIndexes { get; set; @@ -234,10 +234,8 @@ private void LoadFrameworks() } // inspect any TFMs inbox - var frameworkData = FrameworkSet.Load(FrameworkListsPath); - var inboxFrameworks = frameworkData.Frameworks.SelectMany(f => f.Value) - .Where(fx => fx.Assemblies.ContainsKey(PackageId)) - .Select(fx => NuGetFramework.Parse(fx.FrameworkName.FullName)); + var index = PackageIndex.Load(PackageIndexes.Select(pi => pi.GetMetadata("FullPath"))); + var inboxFrameworks = index.GetInboxFrameworks(PackageId).NullAsEmpty(); foreach (var inboxFramework in inboxFrameworks) { diff --git a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/GetInboxFrameworks.cs b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/GetInboxFrameworks.cs index 0fe353e5d9..adc1f54571 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/GetInboxFrameworks.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/GetInboxFrameworks.cs @@ -6,13 +6,14 @@ using Microsoft.Build.Utilities; using System; using System.IO; +using System.Linq; namespace Microsoft.DotNet.Build.Tasks.Packaging { public class GetInboxFrameworks : PackagingTask { [Required] - public string FrameworkListsPath + public ITaskItem[] PackageIndexes { get; set; @@ -40,9 +41,9 @@ public string[] InboxFrameworks public override bool Execute() { - if (null == FrameworkListsPath) + if (PackageIndexes == null && PackageIndexes.Length == 0) { - Log.LogError("FrameworkListsPath argument must be specified"); + Log.LogError("PackageIndexes argument must be specified"); return false; } @@ -52,16 +53,11 @@ public override bool Execute() return false; } - if (!Directory.Exists(FrameworkListsPath)) - { - Log.LogError("FrameworkListsPath '{0}' does not exist", FrameworkListsPath); - return false; - } - Log.LogMessage(LogImportance.Low, "Determining inbox frameworks for {0}, {1}", AssemblyName, AssemblyVersion); + + var index = PackageIndex.Load(PackageIndexes.Select(pi => pi.GetMetadata("FullPath"))); - - InboxFrameworks = Frameworks.GetInboxFrameworksList(FrameworkListsPath, AssemblyName, AssemblyVersion, Log); + InboxFrameworks = index.GetInboxFrameworks(AssemblyName, AssemblyVersion).Select(fx => fx.GetShortFolderName()).ToArray(); return !Log.HasLoggedErrors; } diff --git a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/GetPackageFromModule.cs b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/GetPackageFromModule.cs index 103c31e080..a55b322acf 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/GetPackageFromModule.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/GetPackageFromModule.cs @@ -38,7 +38,7 @@ public class GetPackageFromModule : PackagingTask public override bool Execute() { - Dictionary modulesToPackages; + IDictionary modulesToPackages; if (PackageIndexes != null && PackageIndexes.Length > 0) { diff --git a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/Microsoft.DotNet.Build.Tasks.Packaging.csproj b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/Microsoft.DotNet.Build.Tasks.Packaging.csproj index 5b81d5e7fb..70db49b09f 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/Microsoft.DotNet.Build.Tasks.Packaging.csproj +++ b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/Microsoft.DotNet.Build.Tasks.Packaging.csproj @@ -41,7 +41,6 @@ - @@ -66,14 +65,16 @@ - + PreserveNewest - Designer - + PreserveNewest - + + PreserveNewest + + PreserveNewest diff --git a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/PackageFiles/PackageLibs.targets b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/PackageFiles/PackageLibs.targets index f59417d859..85f54d2d2a 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/PackageFiles/PackageLibs.targets +++ b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/PackageFiles/PackageLibs.targets @@ -1,14 +1,6 @@ - - $(MSBuildThisFileDirectory) - $(MSBuildThisFileDirectory)Generations.json - $(ProjectDir)pkg/Microsoft.NETCore.Platforms/runtime.json - $(MSBuildThisFileDirectory)runtime.json - - - - + diff --git a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/PackageFiles/Packaging.common.targets b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/PackageFiles/Packaging.common.targets new file mode 100644 index 0000000000..f525909914 --- /dev/null +++ b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/PackageFiles/Packaging.common.targets @@ -0,0 +1,42 @@ + + + + <_PackagingCommonTargetsImported>true + + + + $(MSBuildThisFileDirectory) + $(ProjectDir)pkg/Microsoft.NETCore.Platforms/runtime.json + $(MSBuildThisFileDirectory)runtime.json + + + + $(PackageTargetRuntime)-$(PackageTargetRuntimeSuffix) + $(PackageTargetRuntimeSuffix) + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/PackageFiles/Packaging.targets b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/PackageFiles/Packaging.targets index 6384c9c024..4569e8b0f0 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/PackageFiles/Packaging.targets +++ b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/PackageFiles/Packaging.targets @@ -1,5 +1,6 @@ + true - - $(ShouldGenerateNuSpec) $(ShouldGenerateNuSpec) GenerateNuSpec - $(BuildDependsOn);CreateLayout $(BuildDependsOn);CreatePackage $(BuildDependsOn);GetPackageReport;ValidatePackage @@ -635,7 +602,7 @@ + PackageIndexes="@(PackageIndex)"> @@ -729,7 +696,7 @@ compile/runtime. This reduces the noise when consuming our packages in packages.config based projects. --> @@ -738,7 +705,7 @@ @@ -1064,7 +1031,7 @@ Files="@(PackageAsset)" Frameworks="@(DefaultValidateFramework)" RuntimeFile="$(RuntimeIdGraphDefinitionFile)" - FrameworkListsPath="$(FrameworkListsPath)" + PackageIndexes="@(PackageIndex)" ReportFile="$(PackageReportPath)" /> @@ -1097,7 +1064,6 @@ - - - - - - - - - - - - - @@ -1251,42 +1200,16 @@ + + - - - - - - - - - - - - - - <_missingLayoutFiles Include="@(LayoutFiles)" Condition="!Exists('%(FullPath)')" /> - - - - - - + ModuleToPackages="@(ModuleToPackage)" + InboxFrameworkListFolder="$(FrameworkListsFolder)" + InboxFrameworkLayoutFolders="@(FrameworkLayout)" /> diff --git a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/PackageIndex.cs b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/PackageIndex.cs index a995ccb037..50af2c52ed 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/PackageIndex.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/PackageIndex.cs @@ -4,10 +4,15 @@ using Newtonsoft.Json; using Newtonsoft.Json.Converters; +using NuGet.Frameworks; using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Collections; +using Newtonsoft.Json.Linq; +using System.Xml.Linq; namespace Microsoft.DotNet.Build.Tasks.Packaging { @@ -15,9 +20,9 @@ public class PackageIndex { static ConcurrentDictionary s_indexCache = new ConcurrentDictionary(); - public Dictionary Packages { get; set; } = new Dictionary(); + public SortedDictionary Packages { get; set; } = new SortedDictionary(); - public Dictionary ModulesToPackages { get; set; } = new Dictionary(); + public SortedDictionary ModulesToPackages { get; set; } = new SortedDictionary(); public string PreRelease { get; set; } @@ -59,6 +64,7 @@ public static PackageIndex Load(string packageIndexFile) { var serializer = new JsonSerializer(); serializer.Converters.Add(new VersionConverter()); + serializer.Converters.Add(new InboxFrameworksConverter()); var result = serializer.Deserialize(jsonTextReader); result.IndexSources.Add(Path.GetFullPath(packageIndexFile)); return result; @@ -68,7 +74,7 @@ public static PackageIndex Load(string packageIndexFile) public void Save(string path) { string directory = Path.GetDirectoryName(path); - if (!Directory.Exists(directory)) + if (!String.IsNullOrEmpty(directory) && !Directory.Exists(directory)) { Directory.CreateDirectory(directory); } @@ -81,6 +87,7 @@ public void Save(string path) serializer.NullValueHandling = NullValueHandling.Ignore; serializer.DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate; serializer.Converters.Add(new VersionConverter()); + serializer.Converters.Add(new InboxFrameworksConverter()); serializer.Serialize(file, this); } } @@ -168,6 +175,57 @@ public void Merge(PackageIndex other) } } + public void MergeInboxFromLayout(NuGetFramework framework, string layoutDirectory, bool addPackages = true) + { + foreach(var file in Directory.EnumerateFiles(layoutDirectory, "*.dll", SearchOption.AllDirectories)) + { + var assemblyName = Path.GetFileNameWithoutExtension(file); + var assemblyVersion = VersionUtility.GetAssemblyVersion(file); + + TryAddInbox(assemblyName, framework, assemblyVersion, addPackages); + } + } + + public void MergeFrameworkLists(string frameworkListDirectory, bool addPackages = true) + { + foreach (string frameworkDir in Directory.EnumerateDirectories(frameworkListDirectory)) + { + string targetFrameworkMoniker = Path.GetFileName(frameworkDir); + NuGetFramework framework = NuGetFramework.Parse(targetFrameworkMoniker); + foreach (string frameworkListPath in Directory.EnumerateFiles(frameworkDir, "*.xml")) + { + MergeFrameworkList(framework, frameworkListPath, addPackages); + } + } + } + + public void MergeFrameworkList(NuGetFramework framework, string frameworkListPath, bool addPackages = true) + { + XDocument frameworkList = XDocument.Load(frameworkListPath); + foreach (var file in frameworkList.Element("FileList").Elements("File")) + { + string assemblyName = file.Attribute("AssemblyName").Value; + var versionAttribute = file.Attribute("Version"); + Version supportedVersion = (versionAttribute != null) ? new Version(versionAttribute.Value) : VersionUtility.MaxVersion; + + TryAddInbox(assemblyName, framework, supportedVersion, addPackages); + } + } + + private void TryAddInbox(string assemblyName, NuGetFramework framework, Version version, bool addPackages = true) + { + PackageInfo info; + if (Packages.TryGetValue(assemblyName, out info)) + { + info.InboxOn.AddInboxVersion(framework, version); + } + else if (addPackages) + { + Packages[assemblyName] = info = new PackageInfo(); + info.InboxOn.AddInboxVersion(framework, version); + } + } + // helper functions public bool TryGetBaseLineVersion(string packageId, out Version baseLineVersion) { @@ -182,6 +240,77 @@ public bool TryGetBaseLineVersion(string packageId, out Version baseLineVersion) return baseLineVersion != null; } + public IEnumerable GetAlllInboxFrameworks() + { + // very inefficient, included for legacy reasons. + return Packages.Values.SelectMany(info => info.InboxOn.GetInboxFrameworks()).Distinct().ToArray(); + } + + public IEnumerable GetInboxFrameworks(string assemblyName) + { + IEnumerable inboxFrameworks = null; + PackageInfo info; + + if (Packages.TryGetValue(assemblyName, out info)) + { + inboxFrameworks = info.InboxOn.GetInboxFrameworks(); + } + + return inboxFrameworks; + } + public IEnumerable> GetInboxVersions(string assemblyName) + { + IEnumerable> inboxVersions = null; + PackageInfo info; + + if (Packages.TryGetValue(assemblyName, out info)) + { + inboxVersions = info.InboxOn.GetInboxVersions(); + } + + return inboxVersions; + } + + public IEnumerable GetInboxFrameworks(string assemblyName, string assemblyVersionString) + { + Version assemblyVersion = FrameworkUtilities.Ensure4PartVersion(String.IsNullOrEmpty(assemblyVersionString) ? new Version(0, 0, 0, 0) : new Version(assemblyVersionString)); + + return GetInboxFrameworks(assemblyName, assemblyVersion); + } + + public IEnumerable GetInboxFrameworks(string assemblyName, Version assemblyVersion) + { + IEnumerable inboxFrameworks = null; + PackageInfo info; + + if (Packages.TryGetValue(assemblyName, out info)) + { + inboxFrameworks = info.InboxOn.GetInboxVersions().Where(p => p.Value >= assemblyVersion).Select(p => p.Key); + } + + return inboxFrameworks; + } + + public bool IsInbox(string assemblyName, NuGetFramework framework, string assemblyVersionString) + { + Version assemblyVersion = FrameworkUtilities.Ensure4PartVersion(String.IsNullOrEmpty(assemblyVersionString) ? new Version(0, 0, 0, 0) : new Version(assemblyVersionString)); + + return IsInbox(assemblyName, framework, assemblyVersion); + } + + public bool IsInbox(string assemblyName, NuGetFramework framework, Version assemblyVersion) + { + PackageInfo info; + bool isInbox = false; + + if (Packages.TryGetValue(assemblyName, out info)) + { + isInbox = info.InboxOn.IsInbox(framework, assemblyVersion); + } + + return isInbox; + } + public bool IsStable(string packageId, Version packageVersion) { PackageInfo info; @@ -241,10 +370,23 @@ public class PackageInfo public bool ShouldSerializeStableVersions() { return StableVersions.Count > 0; } + /// + /// Minimum version to use when referencing this package. + /// public Version BaselineVersion { get; set; } - public Dictionary AssemblyVersionInPackageVersion { get; set; } = new Dictionary(); + /// + /// Mapping of frameworks which contain this package inbox (or in a framework package) + /// + public InboxFrameworks InboxOn { get; set; } = new InboxFrameworks(); + + public bool ShouldSerializeInboxFrameworkAssemblyVersions() { return InboxOn.Count > 0; } + /// + /// Mapping of assembly version to package version which (first) contains that assembly version. + /// + public SortedDictionary AssemblyVersionInPackageVersion { get; set; } = new SortedDictionary(); + public bool ShouldSerializeAssemblyVersionInPackageVersion() { return AssemblyVersionInPackageVersion.Count > 0; } public string PreRelease { get; set; } @@ -264,6 +406,11 @@ public void Merge(PackageInfo other) PreRelease = other.PreRelease; } + //foreach(var inboxFrameworkAssemblyVersion in other.InboxFrameworkAssemblyVersions) + //{ + + //} + foreach (var assemblyVersionInPackage in other.AssemblyVersionInPackageVersion) { Version otherAssemblyVersion = assemblyVersionInPackage.Key; @@ -294,7 +441,7 @@ public void AddAssemblyVersionInPackage(Version assemblyVersion, Version package // use the lowest stable package version if ((updateStable && !existingStable) || // update to stable from unstable (updateStable && existingStable && packageVersion < existingPackageVersion) || // update to lower stable - (!updateStable && !existingStable && packageVersion > existingPackageVersion)) // update to higher non-stable version + (!updateStable && !existingStable && packageVersion > existingPackageVersion)) // update to higher non-stable version { AssemblyVersionInPackageVersion[assemblyVersion] = packageVersion; } @@ -304,6 +451,22 @@ public void AddAssemblyVersionInPackage(Version assemblyVersion, Version package AssemblyVersionInPackageVersion[assemblyVersion] = packageVersion; } } + public Version GetPackageVersionForAssemblyVersion(NuGetFramework framework, Version assemblyVersion) + { + Version packageVersion = null; + + if (assemblyVersion != null) + { + // prefer an explicit mapping + if (!AssemblyVersionInPackageVersion.TryGetValue(assemblyVersion, out packageVersion)) + { + // if not found assume 1:1 with assembly version + packageVersion = VersionUtility.As3PartVersion(assemblyVersion); + } + } + + return packageVersion; + } public Version GetPackageVersionForAssemblyVersion(Version assemblyVersion) { @@ -321,6 +484,232 @@ public Version GetPackageVersionForAssemblyVersion(Version assemblyVersion) return packageVersion; } + } + + public class InboxFrameworks + { + private SortedDictionary> inboxVersions = new SortedDictionary>(); + + public int Count { get { return inboxVersions.Sum(m => m.Value.Count); } } + + public void AddInboxVersion(NuGetFramework framework, Version assemblyVersion) + { + var normalizedFramework = NormalizeFramework(framework); + var frameworkKey = GetFrameworkKey(normalizedFramework); + var frameworkVersion = normalizedFramework.Version; + + SortedDictionary mappings; + if (!inboxVersions.TryGetValue(frameworkKey, out mappings)) + { + inboxVersions[frameworkKey] = mappings = new SortedDictionary(); + } + + if (assemblyVersion == null) + { + // explicitly not supported. + mappings[normalizedFramework.Version] = assemblyVersion; + } + else + { + List redundantMappings = new List(); + + var addMapping = true; + foreach(var mapping in mappings) + { + var existingFrameworkVersion = mapping.Key; + var existingAssemblyVersion = mapping.Value; + + if (existingFrameworkVersion <= frameworkVersion) + { + if (existingAssemblyVersion != null && existingAssemblyVersion >= assemblyVersion) + { + // lower or same framework already maps this or higher version, don't add it + addMapping = false; + } + } + else + { + if (existingAssemblyVersion <= assemblyVersion) + { + // higher framework version with an equal or lower assembly version, remove it. + redundantMappings.Add(mapping.Key); + } + } + } + + if (addMapping) + { + mappings[normalizedFramework.Version] = assemblyVersion; + } + + foreach(var redundantMapping in redundantMappings) + { + mappings.Remove(redundantMapping); + } + } + + + } + + public IEnumerable GetInboxFrameworks() + { + foreach (var framework in inboxVersions) + { + foreach (var frameworkVersion in framework.Value.Keys) + { + var fx = FromFrameworkKeyAndVersion(framework.Key, frameworkVersion); + + yield return fx; + } + } + } + + public IEnumerable> GetInboxVersions() + { + foreach (var framework in inboxVersions) + { + foreach (var frameworkInboxVersionPair in framework.Value) + { + var frameworkVersion = frameworkInboxVersionPair.Key; + var assemblyVersion = frameworkInboxVersionPair.Value; + + var fx = FromFrameworkKeyAndVersion(framework.Key, frameworkVersion); + + yield return new KeyValuePair(fx, assemblyVersion); + } + } + } + + public bool IsInbox(NuGetFramework framework, Version assemblyVersion) + { + var normalizedFramework = NormalizeFramework(framework); + var key = GetFrameworkKey(normalizedFramework); + + SortedDictionary mappings; + if (!inboxVersions.TryGetValue(key, out mappings)) + { + // no inbox info for this framework + return false; + } + + Version assemblyVersionInbox; + if (mappings.TryGetValue(normalizedFramework.Version, out assemblyVersionInbox)) + { + if (assemblyVersionInbox == null) + { + // null entry means it's explicitly not inbox + return false; + } + + // inbox if explict entry is greater than or equal to current + return assemblyVersionInbox >= assemblyVersion; + } + + // find nearest + var compatibleMapping = mappings.LastOrDefault(m => m.Key < normalizedFramework.Version); + if (compatibleMapping.Key == null || compatibleMapping.Value == null) + { + // either no compatible mapping, or compatible mapping explicitly not inbox + return false; + } + + // inbox if compatible entry is greater than or equal to current + return compatibleMapping.Value >= assemblyVersion; + } + + + private static NuGetFramework NormalizeFramework(NuGetFramework framework) + { + if (framework == FrameworkConstants.CommonFrameworks.NetCore50) + { + // normalize netcore50 -> UAP10. + // this permits us to model that netcore50/uap10 should not inherit inbox state from netcore4* + return FrameworkConstants.CommonFrameworks.UAP10; + } + else if (framework == FrameworkConstants.CommonFrameworks.NetCore45) + { + return FrameworkConstants.CommonFrameworks.Win8; + } + else if (framework == FrameworkConstants.CommonFrameworks.NetCore451) + { + return FrameworkConstants.CommonFrameworks.Win81; + } + + return framework; + } + + private static string GetFrameworkKey(NuGetFramework framework) + { + if (!String.IsNullOrEmpty(framework.Profile)) + { + return framework.Framework + "," + framework.Profile; + } + return framework.Framework; + } + + private static NuGetFramework FromFrameworkKeyAndVersion(string key, Version version) + { + var parts = key.Split(','); + + if (parts.Length > 1) + { + return new NuGetFramework(parts[0], version, parts[1]); + } + else + { + return new NuGetFramework(key, version); + } + } + } + + public class InboxFrameworksConverter : JsonConverter + { + private const string AnyVersion = "Any"; + public override bool CanConvert(Type objectType) + { + return objectType == typeof(InboxFrameworks); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + var jobj = JObject.Load(reader); + + var result = new InboxFrameworks(); + foreach (var property in jobj.Properties()) + { + var versionString = property.Value.ToString(); + var version = versionString.Equals(AnyVersion, StringComparison.OrdinalIgnoreCase) ? VersionUtility.MaxVersion : new Version(versionString); + result.AddInboxVersion(NuGetFramework.Parse(property.Name), version); + } + + return result; + } + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + if (value == null) + { + writer.WriteNull(); + } + else if (value is InboxFrameworks) + { + writer.WriteStartObject(); + + foreach(var frameworkPair in ((InboxFrameworks)value).GetInboxVersions()) + { + var shortName = frameworkPair.Key.GetShortFolderName(); + var assemblyVersion = frameworkPair.Value; + var assemblyVersionString = assemblyVersion == VersionUtility.MaxVersion ? AnyVersion : assemblyVersion.ToString(); + writer.WritePropertyName(shortName); + writer.WriteValue(assemblyVersionString); + } + + writer.WriteEndObject(); + } + else + { + throw new JsonSerializationException($"Expected {nameof(InboxFrameworks)} but got {value.GetType()}"); + } + } } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/PromoteDependencies.cs b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/PromoteDependencies.cs index a1774824a4..e5822f6705 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/PromoteDependencies.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/PromoteDependencies.cs @@ -21,17 +21,27 @@ public class PromoteDependencies : PackagingTask { private const string TargetFrameworkMetadataName = "TargetFramework"; + private PackageIndex index; + [Required] public ITaskItem[] Dependencies { get; set; } - + [Required] - public string FrameworkListsPath { get; set; } + public ITaskItem[] PackageIndexes { get; set; } [Output] public ITaskItem[] PromotedDependencies { get; set; } public override bool Execute() { + if (PackageIndexes == null && PackageIndexes.Length == 0) + { + Log.LogError("PackageIndexes argument must be specified"); + return false; + } + + index = PackageIndex.Load(PackageIndexes.Select(pi => pi.GetMetadata("FullPath"))); + List promotedDependencies = new List(); var dependencies = Dependencies.Select(d => new Dependency(d)).ToArray(); @@ -39,9 +49,12 @@ public override bool Execute() var refSets = dependencies.Where(d => d.Id != "_._").Where(d => d.IsReference).GroupBy(d => d.TargetFramework).ToDictionary(g => NuGetFramework.Parse(g.Key), g => g.ToArray()); var refFxs = refSets.Keys.ToArray(); + Log.LogMessage(LogImportance.Low, $"Ref frameworks {string.Join(";", refFxs.Select(f => f.ToString()))}"); + var libSets = dependencies.Where(d => !d.IsReference).GroupBy(d => d.TargetFramework).ToDictionary(g => NuGetFramework.Parse(g.Key), g => g.ToArray()); var libFxs = libSets.Keys.ToArray(); + Log.LogMessage(LogImportance.Low, $"Lib frameworks {string.Join(";", libFxs.Select(f => f.ToString()))}"); if (libFxs.Length > 0) { @@ -64,13 +77,7 @@ public override bool Execute() // find best lib (if any) var nearestRefFx = FrameworkUtilities.GetNearest(libFx, refFxs); - if (nearestRefFx == null && !nearestRefFx.Equals(libFx)) - { - // This should never happen and indicates a bug in the package. If a package contains references, - // all implementations should have an applicable reference assembly. - Log.LogError($"Could not find applicable reference assembly for implementation framework {libFx} from reference frameworks {string.Join(", ", refFxs.Select(f => f.GetShortFolderName()))}"); - } - else + if (nearestRefFx != null && !nearestRefFx.Equals(libFx)) { promotedDependencies.AddRange(CopyDependencies(refSets[nearestRefFx], libFx)); } @@ -86,7 +93,7 @@ private IEnumerable CopyDependencies(IEnumerable dependen { foreach (var dependency in dependencies) { - if (!Frameworks.IsInbox(FrameworkListsPath, targetFramework, dependency.Id, dependency.Version)) + if (!index.IsInbox(dependency.Id, targetFramework, dependency.Version)) { var copiedDepenedency = new TaskItem(dependency.OriginalItem); copiedDepenedency.SetMetadata(TargetFrameworkMetadataName, targetFramework.GetShortFolderName()); diff --git a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/SplitReferences.cs b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/SplitReferences.cs index 6a1976e2b5..b579b4b99c 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/SplitReferences.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/SplitReferences.cs @@ -27,7 +27,7 @@ public string TargetFramework } [Required] - public string FrameworkListsPath + public ITaskItem[] PackageIndexes { get; set; @@ -51,14 +51,21 @@ public override bool Execute() { if (References == null || References.Length == 0) return true; - + + if (PackageIndexes == null && PackageIndexes.Length == 0) + { + Log.LogError("PackageIndexes argument must be specified"); + return false; + } + + var index = PackageIndex.Load(PackageIndexes.Select(pi => pi.GetMetadata("FullPath"))); + Dictionary packageReferences = new Dictionary(); Dictionary assemblyReferences = new Dictionary(); bool referencesMscorlib = false; NuGetFramework targetFx = NuGetFramework.Parse(TargetFramework); - bool isOobFramework = targetFx.Equals(FrameworkConstants.CommonFrameworks.NetCore50) || targetFx.Framework == FrameworkConstants.FrameworkIdentifiers.UAP; foreach (var reference in References) { @@ -66,7 +73,7 @@ public override bool Execute() referencesMscorlib |= referenceName.Equals("mscorlib"); string referenceVersion = reference.GetMetadata("Version"); reference.SetMetadata("TargetFramework", TargetFramework); - if (!string.IsNullOrEmpty(TargetFramework) && !isOobFramework && Frameworks.IsInbox(FrameworkListsPath, TargetFramework, referenceName, referenceVersion)) + if (!string.IsNullOrEmpty(TargetFramework) && index.IsInbox(referenceName, targetFx, referenceVersion)) { AddReference(assemblyReferences, reference); } diff --git a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/UpdatePackageIndex.cs b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/UpdatePackageIndex.cs index a5bc7458d6..8ad37fe491 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/UpdatePackageIndex.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/UpdatePackageIndex.cs @@ -4,6 +4,7 @@ using Microsoft.Build.Framework; using Newtonsoft.Json; +using NuGet.Frameworks; using NuGet.Packaging; using NuGet.Versioning; using System; @@ -59,6 +60,19 @@ public class UpdatePackageIndex : PackagingTask /// public ITaskItem[] PackageFolders { get; set; } + /// + /// Root folder containing subfolders with framework lists for targeting packs + /// Subfolders must be named by TFM. + /// + public ITaskItem InboxFrameworkListFolder { get; set; } + + /// + /// Folder containing dlls that will be considered inbox + /// Identity: path to folder containing dlls + /// TargetFramework: framework which path represents + /// + public ITaskItem[] InboxFrameworkLayoutFolders { get; set; } + /// /// Pre-release version to use for all pre-release packages covered by this index. /// @@ -139,6 +153,23 @@ public override bool Execute() } } + if (InboxFrameworkListFolder != null) + { + index.MergeFrameworkLists(InboxFrameworkListFolder.GetMetadata("FullPath")); + } + + if (InboxFrameworkLayoutFolders != null) + { + foreach(var inboxFrameworkLayoutFolder in InboxFrameworkLayoutFolders) + { + var layoutDirectory = inboxFrameworkLayoutFolder.GetMetadata("FullPath"); + var targetFramework = NuGetFramework.Parse(inboxFrameworkLayoutFolder.GetMetadata("TargetFramework")); + + index.MergeInboxFromLayout(targetFramework, layoutDirectory); + } + } + + if (!String.IsNullOrEmpty(PreRelease)) { index.PreRelease = PreRelease; diff --git a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/ValidatePackage.cs b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/ValidatePackage.cs index 6fae47dc3a..2f49bef53f 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/ValidatePackage.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/ValidatePackage.cs @@ -94,14 +94,6 @@ public string SuppressionFile set; } - - [Required] - public string FrameworkListsPath - { - get; - set; - } - public bool SkipGenerationCheck { get; @@ -160,10 +152,9 @@ public string ReportFile /// private Dictionary> _suppressions; private Dictionary _frameworks; - private FrameworkSet _frameworkSet; + private PackageIndex _index; private PackageReport _report; private string _generationIdentifier = FrameworkConstants.FrameworkIdentifiers.NetStandard; - private static Version s_maxVersion = new Version(int.MaxValue, int.MaxValue, int.MaxValue, int.MaxValue); public override bool Execute() { @@ -467,9 +458,10 @@ private void ValidateIndex() // not in the native module map, see if any of the modules in this package are present // (with a different package, as would be the case for runtime-specific packages) var moduleNames = allDlls.Select(d => Path.GetFileNameWithoutExtension(d.LocalPath)); - if (moduleNames.Any() && !moduleNames.Any(m => index.ModulesToPackages.ContainsKey(m))) + var missingModuleNames = moduleNames.Where(m => !index.ModulesToPackages.ContainsKey(m)); + if (missingModuleNames.Any()) { - Log.LogError($"PackageIndex from {String.Join(", ", PackageIndexes.Select(i => i.ItemSpec))} is missing ModulesToPackages entry(s) for {String.Join(", ", allDlls)} to package {PackageId}. Please add a an entry for the appropriate package."); + Log.LogError($"PackageIndex from {String.Join(", ", PackageIndexes.Select(i => i.ItemSpec))} is missing ModulesToPackages entry(s) for {String.Join(", ", missingModuleNames)} to package {PackageId}. Please add a an entry for the appropriate package."); } } @@ -709,51 +701,46 @@ private void LoadSupport() // determine which Frameworks should support inbox - _frameworkSet = FrameworkSet.Load(FrameworkListsPath); - foreach (IEnumerable inboxFxGroup in _frameworkSet.Frameworks.Values) + _index = PackageIndex.Load(PackageIndexes.Select(pi => pi.GetMetadata("FullPath"))); + foreach (var inboxPair in _index.GetInboxVersions(ContractName).NullAsEmpty()) { - foreach (Framework inboxFx in inboxFxGroup) - { - // get currently supported version to see if we have OOB'ed it - Version inboxVersion = null; - inboxFx.Assemblies.TryGetValue(ContractName, out inboxVersion); + var fx = inboxPair.Key; + var inboxVersion = inboxPair.Value; - if (inboxVersion != null) + if (inboxVersion != null) + { + ValidationFramework validationFramework = null; + if (_frameworks.TryGetValue(fx, out validationFramework)) { - NuGetFramework fx = FrameworkUtilities.ParseNormalized(inboxFx.ShortName); - ValidationFramework validationFramework = null; - if (_frameworks.TryGetValue(fx, out validationFramework)) - { - Version supportedVersion = validationFramework.SupportedVersion; - - if (supportedVersion != null && - (supportedVersion.Major > inboxVersion.Major || - (supportedVersion.Major == inboxVersion.Major && supportedVersion.Minor > inboxVersion.Minor))) - { - // Higher major.minor - Log.LogMessage(LogImportance.Low, $"Framework {fx} supported {ContractName} as inbox but the current supported version {supportedVersion} is higher in major.minor than inbox version {inboxVersion}. Assuming out of box."); - continue; - } - else if (supportedVersion != null && supportedVersion < inboxVersion && inboxVersion != s_maxVersion) - { - // Lower version - Log.LogError($"Framework {fx} supports {ContractName} as inbox but the current supported version {supportedVersion} is lower than the inbox version {inboxVersion}"); - } + Version supportedVersion = validationFramework.SupportedVersion; - // equal major.minor, build.revision difference is permitted, prefer the version listed by ContractSupport item + if (supportedVersion != null && + (supportedVersion.Major > inboxVersion.Major || + (supportedVersion.Major == inboxVersion.Major && supportedVersion.Minor > inboxVersion.Minor))) + { + // Higher major.minor + Log.LogMessage(LogImportance.Low, $"Framework {fx} supported {ContractName} as inbox but the current supported version {supportedVersion} is higher in major.minor than inbox version {inboxVersion}. Assuming out of box."); + continue; } - - if (validationFramework == null) + else if (supportedVersion != null && supportedVersion < inboxVersion && inboxVersion != VersionUtility.MaxVersion) { - // we may not be explicitly validating for this framework so add it to validate inbox assets. - _frameworks[fx] = validationFramework = new ValidationFramework(fx) - { - SupportedVersion = inboxVersion - }; + // Lower version + Log.LogError($"Framework {fx} supports {ContractName} as inbox but the current supported version {supportedVersion} is lower than the inbox version {inboxVersion}"); } - validationFramework.IsInbox = true; + // equal major.minor, build.revision difference is permitted, prefer the version listed by ContractSupport item } + + if (validationFramework == null) + { + // we may not be explicitly validating for this framework so add it to validate inbox assets. + _frameworks[fx] = validationFramework = new ValidationFramework(fx) + { + SupportedVersion = inboxVersion + }; + } + + validationFramework.IsInbox = true; } } @@ -763,7 +750,7 @@ private void LoadSupport() // their own implementation via a lineup/runtime.json. // only consider frameworks that support the contract at a specific version - var inferFrameworks = _frameworks.Values.Where(fx => fx.SupportedVersion != null && fx.SupportedVersion != s_maxVersion).ToArray(); + var inferFrameworks = _frameworks.Values.Where(fx => fx.SupportedVersion != null && fx.SupportedVersion != VersionUtility.MaxVersion).ToArray(); var genVersionSuppression = GetSuppressionValues(Suppression.PermitPortableVersionMismatch) ?? new HashSet(); var inferNETStandardSuppression = GetSuppressionValues(Suppression.SuppressNETStandardInference) ?? new HashSet(); @@ -801,12 +788,6 @@ private void LoadSupport() } } - private string GetVersionString(Version version) - { - // normalize to API version - return version == s_maxVersion ? "Any" : _frameworkSet.GetApiVersion(ContractName, version)?.ToString(); - } - private class ValidationFramework { private static readonly string[] s_nullRidList = new string[] { null }; diff --git a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/VersionUtility.cs b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/VersionUtility.cs index 59a67d089c..020c1f4688 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Packaging/src/VersionUtility.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Packaging/src/VersionUtility.cs @@ -11,6 +11,8 @@ namespace Microsoft.DotNet.Build.Tasks.Packaging { internal static class VersionUtility { + public static readonly Version MaxVersion = new Version(int.MaxValue, int.MaxValue, int.MaxValue, int.MaxValue); + public static bool IsCompatibleApiVersion(Version referenceVersion, Version definitionVersion) { return (referenceVersion.Major == definitionVersion.Major && diff --git a/src/Microsoft.DotNet.Build.Tasks.Packaging/test/CreateTrimDependencyGroupsTests.cs b/src/Microsoft.DotNet.Build.Tasks.Packaging/test/CreateTrimDependencyGroupsTests.cs index 6912aa6fa8..0a9a639e04 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Packaging/test/CreateTrimDependencyGroupsTests.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Packaging/test/CreateTrimDependencyGroupsTests.cs @@ -16,11 +16,21 @@ public class CreateTrimDependencyGroupsTests { private Log _log; private TestBuildEngine _engine; + private ITaskItem[] packageIndexes; + + private const string FrameworkListsPath = "FrameworkLists"; public CreateTrimDependencyGroupsTests(ITestOutputHelper output) { _log = new Log(output); _engine = new TestBuildEngine(_log); + + + var packageIndexPath = $"packageIndex.{Guid.NewGuid()}.json"; + PackageIndex index = new PackageIndex(); + index.MergeFrameworkLists(FrameworkListsPath); + index.Save(packageIndexPath); + packageIndexes = new[] { new TaskItem(packageIndexPath) }; } [Fact] @@ -74,14 +84,13 @@ public void NoAdditionalDependenciesForPlaceholders() CreateDependencyItem(@"System.Globalization", "4.0.0", "netstandard1.0"), CreateDependencyItem(@"System.Threading", "4.0.0", "netstandard1.0") }; - string frameworkListsPath = "FrameworkLists"; CreateTrimDependencyGroups task = new CreateTrimDependencyGroups() { BuildEngine = _engine, Files = files, Dependencies = dependencies, - FrameworkListsPath = frameworkListsPath + PackageIndexes = packageIndexes }; _log.Reset(); @@ -114,14 +123,13 @@ public void NoPlaceholders() CreateDependencyItem(@"System.Globalization", "4.0.0", "netstandard1.0"), CreateDependencyItem(@"System.Threading", "4.0.0", "netstandard1.0") }; - string frameworkListsPath = "FrameworkLists"; CreateTrimDependencyGroups task = new CreateTrimDependencyGroups() { BuildEngine = _engine, Files = files, Dependencies = dependencies, - FrameworkListsPath = frameworkListsPath + PackageIndexes = packageIndexes }; _log.Reset(); @@ -194,14 +202,13 @@ public void MultiGeneration() CreateDependencyItem(@"System.Collections.Immutable", "4.0.20", "netstandard1.3"), CreateDependencyItem(@"System.Runtime", "4.0.20", ".NETCore50") }; - string frameworkListsPath = "FrameworkLists"; CreateTrimDependencyGroups task = new CreateTrimDependencyGroups() { BuildEngine = _engine, Files = files, Dependencies = dependencies, - FrameworkListsPath = frameworkListsPath + PackageIndexes = packageIndexes }; _log.Reset(); @@ -245,14 +252,13 @@ public void NotSupported() CreateDependencyItem(@"System.Runtime.Handles", "4.0.0", "netcoreapp1.0"), CreateDependencyItem(@"System.Threading", "4.0.10", "netcoreapp1.0") }; - string frameworkListsPath = "FrameworkLists"; CreateTrimDependencyGroups task = new CreateTrimDependencyGroups() { BuildEngine = _engine, Files = files, Dependencies = dependencies, - FrameworkListsPath = frameworkListsPath + PackageIndexes = packageIndexes }; _log.Reset(); @@ -292,14 +298,13 @@ public void AddInboxFrameworkGroupsAndDependencies() CreateDependencyItem(@"System.Collections.Immutable", "1.1.37", "netstandard1.1"), CreateDependencyItem(@"System.Collections.Immutable", "1.1.37", "portable-net45+win80") }; - string frameworkListsPath = "FrameworkLists"; CreateTrimDependencyGroups task = new CreateTrimDependencyGroups() { BuildEngine = _engine, Files = files, Dependencies = dependencies, - FrameworkListsPath = frameworkListsPath + PackageIndexes = packageIndexes }; _log.Reset(); diff --git a/src/nuget/Microsoft.DotNet.BuildTools.nuspec b/src/nuget/Microsoft.DotNet.BuildTools.nuspec index 2e1bbf9e2e..01a69c9b37 100644 --- a/src/nuget/Microsoft.DotNet.BuildTools.nuspec +++ b/src/nuget/Microsoft.DotNet.BuildTools.nuspec @@ -47,7 +47,6 @@ -