From eef36da4a26be31905754be0aad7378019c6af18 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Mon, 29 Apr 2024 14:37:39 +0200 Subject: [PATCH 01/10] [msbuild] Refactor out function in CollectBundleResource to make code reuse possible. --- .../Tasks/CollectBundleResources.cs | 69 +++++++++++-------- 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/CollectBundleResources.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/CollectBundleResources.cs index 6efcb5dfa9d3..2b676f32a3f4 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/CollectBundleResources.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/CollectBundleResources.cs @@ -68,38 +68,9 @@ bool ExecuteImpl () var bundleResources = new List (); foreach (var item in BundleResources) { - // Skip anything with the PublishFolderType metadata, these are copied directly to the ResolvedFileToPublish item group instead. - var publishFolderType = item.GetMetadata ("PublishFolderType"); - if (!string.IsNullOrEmpty (publishFolderType)) + if (!TryCreateItemWithLogicalName (this, item, out var bundleResource)) continue; - var logicalName = BundleResource.GetLogicalName (this, item); - // We need a physical path here, ignore the Link element - var path = item.GetMetadata ("FullPath"); - - if (!File.Exists (path)) { - Log.LogError (MSBStrings.E0099, logicalName, path); - continue; - } - - if (logicalName.StartsWith (".." + Path.DirectorySeparatorChar, StringComparison.Ordinal)) { - Log.LogError (null, null, null, item.ItemSpec, 0, 0, 0, 0, MSBStrings.E0100, logicalName); - continue; - } - - if (logicalName == "Info.plist") { - Log.LogWarning (null, null, null, item.ItemSpec, 0, 0, 0, 0, MSBStrings.E0101); - continue; - } - - if (BundleResource.IsIllegalName (logicalName, out var illegal)) { - Log.LogError (null, null, null, item.ItemSpec, 0, 0, 0, 0, MSBStrings.E0102, illegal); - continue; - } - - var bundleResource = new TaskItem (item); - bundleResource.SetMetadata ("LogicalName", logicalName); - bool optimize = false; if (CanOptimize (item.ItemSpec)) { @@ -127,6 +98,44 @@ bool ExecuteImpl () return !Log.HasLoggedErrors; } + public static bool TryCreateItemWithLogicalName (T task, ITaskItem item, [NotNullWhen (true)] out TaskItem? itemWithLogicalName) where T : Task, IHasProjectDir, IHasResourcePrefix, IHasSessionId + { + itemWithLogicalName = null; + + // Skip anything with the PublishFolderType metadata, these are copied directly to the ResolvedFileToPublish item group instead. + var publishFolderType = item.GetMetadata ("PublishFolderType"); + if (!string.IsNullOrEmpty (publishFolderType)) + return false; + + var logicalName = BundleResource.GetLogicalName (task, item); + // We need a physical path here, ignore the Link element + var path = item.GetMetadata ("FullPath"); + + if (!File.Exists (path)) { + task.Log.LogError (MSBStrings.E0099, logicalName, path); + return false; + } + + if (logicalName.StartsWith (".." + Path.DirectorySeparatorChar, StringComparison.Ordinal)) { + task.Log.LogError (null, null, null, item.ItemSpec, 0, 0, 0, 0, MSBStrings.E0100, logicalName); + return false; + } + + if (logicalName == "Info.plist") { + task.Log.LogWarning (null, null, null, item.ItemSpec, 0, 0, 0, 0, MSBStrings.E0101); + return false; + } + + if (BundleResource.IsIllegalName (logicalName, out var illegal)) { + task.Log.LogError (null, null, null, item.ItemSpec, 0, 0, 0, 0, MSBStrings.E0102, illegal); + return false; + } + + itemWithLogicalName = new TaskItem (item); + itemWithLogicalName.SetMetadata ("LogicalName", logicalName); + return true; + } + public void Cancel () { if (ShouldExecuteRemotely ()) From e12dbbd1e10e50001f58f171e6f9a61cac87dd86 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Thu, 1 Feb 2024 08:52:31 +0100 Subject: [PATCH 02/10] [msbuild] Add support for bundling original resources in libraries. Fixes #19028. If a library references resources, until now we've pre-compile/pre-processed some of those before embedding them the library. This applies to resources of the following item groups: * AtlasTexture * BundleResource * Collada * CoreMLModel * ImageAsset * InterfaceDefinition * SceneKitAsset However, pre-processing resources as a few problems: * It requires a native (Xcode) toolchain. * This is unfortunate when building from Windows: the current approach is that when building a library as a referenced project, the remoting part is skipped, so all such resources are just dropped. * It also means building on Linux doesn't work. * It makes it impossible to merge resources with the same name, if we wanted to do that. So I'm adding support for bundling the original resources in library projects. This is enabled using the MSBuild property `BundleOriginalResources=true`, which is turned off by default for .NET 9 and turned on by default for .NET 10+. Fixes https://github.com/xamarin/xamarin-macios/issues/19028. --- docs/build-apps/build-properties.md | 25 ++ dotnet/targets/Xamarin.Shared.Sdk.targets | 1 + .../MSBStrings.resx | 4 + .../Tasks/CollectBundleResources.cs | 4 + .../Tasks/CollectPackLibraryResources.cs | 86 +++++ .../Tasks/PackLibraryResources.cs | 24 ++ .../Tasks/UnpackLibraryResources.cs | 347 ++++++++++++++---- msbuild/Xamarin.Shared/Xamarin.Shared.props | 6 + msbuild/Xamarin.Shared/Xamarin.Shared.targets | 94 ++++- .../Xamarin.Messaging.Apple.targets | 1 + .../Xamarin.iOS.Common.After.targets | 2 +- 11 files changed, 500 insertions(+), 94 deletions(-) create mode 100644 msbuild/Xamarin.MacDev.Tasks/Tasks/CollectPackLibraryResources.cs diff --git a/docs/build-apps/build-properties.md b/docs/build-apps/build-properties.md index c16c47be8c9c..89365a202a90 100644 --- a/docs/build-apps/build-properties.md +++ b/docs/build-apps/build-properties.md @@ -61,6 +61,31 @@ The directory to where the `bgen` ([BGenToolExe](#BGenToolExe)) is located. The default behavior is to use the `bgen` tool shipped with our workload. +## BundleOriginalResources + +This property determines whether resources are compiled before being embedded +into library projects, or if the original (uncompiled) version is embedded. + +Historically resources have been compiled before being embedded into library +projects, but this requires having Xcode available, which has a few drawbacks: + +* It slows down remote builds on Windows. +* It won't work when building locally on Windows, and neither on any other + platform except macOS. +* Resources are compiled using the current available Xcode, which may not have + the same features as a potentially newer Xcode available when the library in + question is consumed. +* It makes it impossible to have a whole-program view of all the resources + when building an app, which is necessary to detect clashing resources. + +As such, we've added supported for embedding the original resources into +libraries. This will be opt-in in .NET 9, but opt-out starting in .NET 10. + +Default value: `false` in .NET 9, `true` in .NET 10+. + +Note: please file an issue if you find that you need to disable this feature, +as it's possible we'll remove the option to disable it at some point. + ## DittoPath The full path to the `ditto` executable. diff --git a/dotnet/targets/Xamarin.Shared.Sdk.targets b/dotnet/targets/Xamarin.Shared.Sdk.targets index 1504cb8c6a9f..21dfee3fabc7 100644 --- a/dotnet/targets/Xamarin.Shared.Sdk.targets +++ b/dotnet/targets/Xamarin.Shared.Sdk.targets @@ -233,6 +233,7 @@ _ErrorRuntimeIdentifiersClash; + BuildOnlySettings; _CollectBundleResources; _RunRidSpecificBuild; _DetectAppManifest; diff --git a/msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx b/msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx index fe78bce6a182..21c4a18f256d 100644 --- a/msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx +++ b/msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx @@ -1654,4 +1654,8 @@ {1}: the exit code of a process + + + Unknown resource type: {1}. + diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/CollectBundleResources.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/CollectBundleResources.cs index 2b676f32a3f4..a6fab5c485c3 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/CollectBundleResources.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/CollectBundleResources.cs @@ -32,6 +32,8 @@ public class CollectBundleResources : XamarinTask, ICancelableTask, IHasProjectD [Output] public ITaskItem [] BundleResourcesWithLogicalNames { get; set; } = Array.Empty (); + public ITaskItem [] UnpackedResources { get; set; } = Array.Empty (); + #endregion static bool CanOptimize (string path) @@ -93,6 +95,8 @@ bool ExecuteImpl () bundleResources.Add (bundleResource); } + bundleResources.AddRange (UnpackedResources); + BundleResourcesWithLogicalNames = bundleResources.ToArray (); return !Log.HasLoggedErrors; diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/CollectPackLibraryResources.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/CollectPackLibraryResources.cs new file mode 100644 index 000000000000..b88bcdb32914 --- /dev/null +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/CollectPackLibraryResources.cs @@ -0,0 +1,86 @@ +using System; +using System.IO; +using System.Collections.Generic; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Xamarin.Localization.MSBuild; +using Xamarin.Messaging.Build.Client; + +namespace Xamarin.MacDev.Tasks { + // This task will collect several item groups with various types of assets/resources, + // add/compute the LogicalName value for each of them, and then add them to the + // ItemsWithLogicalNames item group. The items in this item group will have the + // 'OriginalItemGroup' metadata set indicating where they came from. + public class CollectPackLibraryResources : XamarinTask, IHasProjectDir, IHasResourcePrefix { + #region Inputs + + public ITaskItem [] AtlasTextures { get; set; } = Array.Empty (); + + public ITaskItem [] BundleResources { get; set; } = Array.Empty (); + + public ITaskItem [] ImageAssets { get; set; } = Array.Empty (); + + public ITaskItem [] InterfaceDefinitions { get; set; } = Array.Empty (); + + public ITaskItem [] ColladaAssets { get; set; } = Array.Empty (); + + public ITaskItem [] CoreMLModels { get; set; } = Array.Empty (); + + public ITaskItem [] PartialAppManifests { get; set; } = Array.Empty (); + + public ITaskItem [] SceneKitAssets { get; set; } = Array.Empty (); + + [Required] + public string ProjectDir { get; set; } = string.Empty; + + [Required] + public string ResourcePrefix { get; set; } = string.Empty; + + #endregion + + #region Outputs + + // These items will have the following metadata set: + // * LogicalName + // * OriginalItemGroup: the name of the originating item group + [Output] + public ITaskItem [] ItemsWithLogicalNames { get; set; } = Array.Empty (); + + #endregion + + public override bool Execute () + { + var prefixes = BundleResource.SplitResourcePrefixes (ResourcePrefix); + var rv = new List (); + + var resources = new [] { + new { Name = "AtlasTexture", Items = AtlasTextures }, + new { Name = "BundleResource", Items = BundleResources }, + new { Name = "Collada", Items = ColladaAssets }, + new { Name = "CoreMLModel", Items = CoreMLModels }, + new { Name = "ImageAsset", Items = ImageAssets }, + new { Name = "InterfaceDefinition", Items = InterfaceDefinitions }, + new { Name = "PartialAppManifest", Items = PartialAppManifests }, + new { Name = "SceneKitAsset", Items = SceneKitAssets }, + }; + + foreach (var kvp in resources) { + var itemName = kvp.Name; + var items = kvp.Items; + + foreach (var item in items) { + if (!CollectBundleResources.TryCreateItemWithLogicalName (this, item, out var itemWithLogicalName)) + continue; + + itemWithLogicalName.SetMetadata ("OriginalItemGroup", itemName); + rv.Add (itemWithLogicalName); + } + } + + ItemsWithLogicalNames = rv.ToArray (); + + return !Log.HasLoggedErrors; + } + } +} diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/PackLibraryResources.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/PackLibraryResources.cs index 163a08b432cd..afb93f312004 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/PackLibraryResources.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/PackLibraryResources.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Text; using System.Collections.Generic; @@ -17,6 +18,8 @@ public class PackLibraryResources : XamarinTask, ITaskCallback, ICancelableTask public ITaskItem [] BundleResourcesWithLogicalNames { get; set; } = Array.Empty (); + public ITaskItem [] BundleOriginalResourcesWithLogicalNames { get; set; } = Array.Empty (); + #endregion #region Outputs @@ -97,11 +100,32 @@ public override bool Execute () results.Add (embedded); } + foreach (var item in BundleOriginalResourcesWithLogicalNames) { + var originalItemGroup = item.GetMetadata ("OriginalItemGroup"); + if (!TryGetMangledLogicalName (item, originalItemGroup, out var mangledLogicalName)) + continue; + var embedded = new TaskItem (item); + embedded.SetMetadata ("LogicalName", mangledLogicalName); + results.Add (embedded); + } + EmbeddedResources = results.ToArray (); return !Log.HasLoggedErrors; } + bool TryGetMangledLogicalName (ITaskItem item, string itemName, [NotNullWhen (true)] out string? mangled) + { + var logicalName = item.GetMetadata ("LogicalName"); + if (string.IsNullOrEmpty (logicalName)) { + Log.LogError (null, null, null, item.ItemSpec, 0, 0, 0, 0, MSBStrings.E0161); + mangled = null; + return false; + } + mangled = "__" + Prefix + "_item_" + itemName + "_" + EscapeMangledResource (logicalName); + return true; + } + public void Cancel () { if (ShouldExecuteRemotely ()) diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/UnpackLibraryResources.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/UnpackLibraryResources.cs index 558c27d52b3c..0bb15a4cf5f8 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/UnpackLibraryResources.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/UnpackLibraryResources.cs @@ -1,4 +1,5 @@ using System; +using System.Collections; using System.IO; using System.Linq; using System.Reflection; @@ -47,8 +48,40 @@ public class UnpackLibraryResources : XamarinTask, ITaskCallback, ICancelableTas [Output] public ITaskItem [] UnpackedResources { get; set; } = Array.Empty (); + [Output] + public ITaskItem [] AtlasTextures { get; set; } = Array.Empty (); + + [Output] + public ITaskItem [] ColladaAssets { get; set; } = Array.Empty (); + + [Output] + public ITaskItem [] CoreMLModels { get; set; } = Array.Empty (); + + [Output] + public ITaskItem [] ImageAssets { get; set; } = Array.Empty (); + + [Output] + public ITaskItem [] InterfaceDefinitions { get; set; } = Array.Empty (); + + [Output] + public ITaskItem [] PartialAppManifests { get; set; } = Array.Empty (); + + [Output] + public ITaskItem [] SceneKitAssets { get; set; } = Array.Empty (); + #endregion + enum ResourceType { + AtlasTexture, + BundleResource, + ColladaAsset, + CoreMLModel, + ImageAsset, + InterfaceDefinition, + PartialAppManifest, + SceneKitAsset, + } + public override bool Execute () { if (ShouldExecuteRemotely ()) { @@ -67,7 +100,14 @@ public override bool Execute () return result; } - var results = new List (); + var bundleResources = new List (); + var atlasTextures = new List (); + var colladaAssets = new List (); + var coreMLModels = new List (); + var imageAssets = new List (); + var interfaceDefinitions = new List (); + var partialAppManifests = new List (); + var sceneKitAssets = new List (); foreach (var asm in ReferencedLibraries) { // mscorlib.dll was not coming out with ResolvedFrom == {TargetFrameworkDirectory} @@ -78,15 +118,50 @@ public override bool Execute () var perAssemblyOutputPath = Path.Combine (IntermediateOutputPath, "unpack", asm.GetMetadata ("Filename")); var extracted = ExtractContentAssembly (asm.ItemSpec, perAssemblyOutputPath); - results.AddRange (extracted); - - var itemsFile = asm.GetMetadata ("ItemsFile"); - itemsFile = itemsFile.Replace ('\\', Path.DirectorySeparatorChar); - WriteItemsToFile.Write (this, itemsFile, extracted, "_BundleResourceWithLogicalName", true, true); + foreach (var tuple in extracted) { + var resourceType = tuple.Type; + var item = tuple.Item; + switch (resourceType) { + case ResourceType.AtlasTexture: + atlasTextures.Add (item); + break; + case ResourceType.BundleResource: + bundleResources.Add (item); + break; + case ResourceType.ColladaAsset: + colladaAssets.Add (item); + break; + case ResourceType.CoreMLModel: + coreMLModels.Add (item); + break; + case ResourceType.ImageAsset: + imageAssets.Add (item); + break; + case ResourceType.InterfaceDefinition: + interfaceDefinitions.Add (item); + break; + case ResourceType.PartialAppManifest: + partialAppManifests.Add (item); + break; + case ResourceType.SceneKitAsset: + sceneKitAssets.Add (item); + break; + default: + Log.LogError (MSBStrings.E7134 /* Unknown resource type: {1}. */, resourceType); + break; + } + } } } - BundleResourcesWithLogicalNames = results.ToArray (); + BundleResourcesWithLogicalNames = bundleResources.ToArray (); + AtlasTextures = atlasTextures.ToArray (); + ColladaAssets = colladaAssets.ToArray (); + CoreMLModels = coreMLModels.ToArray (); + ImageAssets = imageAssets.ToArray (); + InterfaceDefinitions = interfaceDefinitions.ToArray (); + PartialAppManifests = partialAppManifests.ToArray (); + SceneKitAssets = sceneKitAssets.ToArray (); UnpackedResources = unpackedResources.ToArray (); return !Log.HasLoggedErrors; @@ -102,22 +177,89 @@ bool IsFrameworkAssembly (ITaskItem asm) return false; } - List ExtractContentAssembly (string assembly, string intermediatePath) - { - var rv = new List (); + class AssemblyResource { + ResourceExtractor Extractor; + public ManifestResource ManifestResource; + public string Name; + public AssemblyResource (ResourceExtractor extractor, ManifestResource manifestResource, string name) + { + Extractor = extractor; + ManifestResource = manifestResource; + Name = name; + } + } - if (!File.Exists (assembly)) { - Log.LogMessage (MessageImportance.Low, $"Not inspecting assembly because it doesn't exist: {assembly}"); - return rv; + class ResourceExtractor : IDisposable, IEnumerable { + string assembly; + FileStream? peStream; + PEReader? peReader; + MetadataReader? metadataReader; + TaskLoggingHelper log; + + public ResourceExtractor (TaskLoggingHelper log, string assembly) + { + this.log = log; + this.assembly = assembly; } - try { - var asmWriteTime = File.GetLastWriteTimeUtc (assembly); - using var peStream = File.OpenRead (assembly); - using var peReader = new PEReader (peStream); - var metadataReader = PEReaderExtensions.GetMetadataReader (peReader); - Log.LogMessage (MessageImportance.Low, $"Inspecting resources in assembly {assembly}"); - foreach (var manifestResourceHandle in metadataReader.ManifestResources) { + public void Dispose () + { + peReader?.Dispose (); + peReader = null; + peStream?.Dispose (); + peStream = null; + } + + public void WriteResourceTo (AssemblyResource resource, string path) + { + if (peReader is null) + throw new ObjectDisposedException ("this"); + + Directory.CreateDirectory (Path.GetDirectoryName (path)); + + var manifestResource = resource.ManifestResource; + var resourceDirectory = peReader.GetSectionData (peReader.PEHeaders.CorHeader!.ResourcesDirectory.RelativeVirtualAddress); + var reader = resourceDirectory.GetReader ((int) manifestResource.Offset, resourceDirectory.Length - (int) manifestResource.Offset); + var length = reader.ReadUInt32 (); + if (length > reader.RemainingBytes) + throw new BadImageFormatException (); +#if NET + using var fs = new FileStream (path, FileMode.Create, FileAccess.Write, FileShare.Read); + unsafe { + var span = new ReadOnlySpan (reader.CurrentPointer, (int) length); + fs.Write (span); + } +#else + var buffer = new byte [4096]; + using var fs = new FileStream (path, FileMode.Create, FileAccess.Write, FileShare.Read, buffer.Length); + var left = (int) length; + while (left > 0) { + var read = Math.Min (left, buffer.Length); + reader.ReadBytes (read, buffer, 0); + fs.Write (buffer, 0, read); + left -= read; + } +#endif + } + + IEnumerator IEnumerable.GetEnumerator () + { + return ((IEnumerable) this).GetEnumerator (); + } + + IEnumerator IEnumerable.GetEnumerator () + { + if (!File.Exists (assembly)) + yield break; + + if (peStream is null) { + peStream = File.OpenRead (assembly); + peReader = new PEReader (peStream); + metadataReader = PEReaderExtensions.GetMetadataReader (peReader); + log.LogMessage (MessageImportance.Low, $"Inspecting resources in assembly {assembly}"); + } + + foreach (var manifestResourceHandle in metadataReader!.ManifestResources) { var manifestResource = metadataReader.GetManifestResource (manifestResourceHandle); if (!manifestResource.Implementation.IsNil) continue; // embedded resources have Implementation.IsNil = true, and those are the ones we care about @@ -126,62 +268,125 @@ List ExtractContentAssembly (string assembly, string intermediatePath if (string.IsNullOrEmpty (name)) continue; + yield return new AssemblyResource (this, manifestResource, name); + } + } + } + + class AssemblyContentResource { + public ResourceType Type; + public ITaskItem Item; + public AssemblyContentResource (ResourceType type, ITaskItem item) + { + Type = type; + Item = item; + } + } + + List ExtractContentAssembly (string assembly, string intermediatePath) + { + var rv = new List (); + + if (!File.Exists (assembly)) { + Log.LogMessage (MessageImportance.Low, $"Not inspecting assembly because it doesn't exist: {assembly}"); + return rv; + } + + var asmWriteTime = File.GetLastWriteTimeUtc (assembly); + using var extractor = new ResourceExtractor (Log, assembly); + + try { + // Log.LogMessage (MessageImportance.Low, " Searching resources in assembly: {0}", assembly); + foreach (var embedded in extractor) { string rpath; - if (name.StartsWith ("__" + Prefix + "_content_", StringComparison.Ordinal)) { - var mangled = name.Substring (("__" + Prefix + "_content_").Length); - rpath = UnmangleResource (mangled); - } else if (name.StartsWith ("__" + Prefix + "_page_", StringComparison.Ordinal)) { - var mangled = name.Substring (("__" + Prefix + "_page_").Length); - rpath = UnmangleResource (mangled); - } else { + var resourceName = embedded.Name; + var startsWith = "__" + Prefix + "_"; + if (!resourceName.StartsWith (startsWith, StringComparison.Ordinal)) { + Log.LogMessage (MessageImportance.Low, $" Not applicable resource (does not match prefix '{startsWith}'): {resourceName}"); + continue; + } + + var underscoreIndex = resourceName.IndexOf ('_', startsWith.Length); + if (underscoreIndex == -1) { + Log.LogMessage (MessageImportance.Low, $" Not applicable resource (no content type found): {resourceName}"); + continue; + } + var contentType = resourceName.Substring (startsWith.Length, underscoreIndex - startsWith.Length); + var contentValue = resourceName.Substring (underscoreIndex + 1); + ResourceType resourceType; + string itemType; + switch (contentType) { + case "content": + case "page": + rpath = UnmangleResource (contentValue); + resourceType = ResourceType.BundleResource; + itemType = contentType; + break; + case "item": + var itemUnderscoreIndex = contentValue.IndexOf ('_'); + if (itemUnderscoreIndex == -1) { + Log.LogMessage (MessageImportance.Low, $" Not applicable resource (no item type in '{contentValue}'): {resourceName}"); + continue; + } + itemType = contentValue.Substring (0, itemUnderscoreIndex); + var itemValue = contentValue.Substring (itemUnderscoreIndex + 1); + rpath = UnmangleResource (itemValue); + switch (itemType) { + case "AtlasTexture": + resourceType = ResourceType.AtlasTexture; + break; + case "BundleResource": + resourceType = ResourceType.BundleResource; + break; + case "Collada": + resourceType = ResourceType.ColladaAsset; + break; + case "CoreMLModel": + resourceType = ResourceType.CoreMLModel; + break; + case "ImageAsset": + resourceType = ResourceType.ImageAsset; + break; + case "InterfaceDefinition": + resourceType = ResourceType.InterfaceDefinition; + break; + case "PartialAppManifest": + resourceType = ResourceType.PartialAppManifest; + break; + case "SceneKitAsset": + resourceType = ResourceType.SceneKitAsset; + break; + default: + Log.LogMessage (MessageImportance.Low, $" Not applicable resource (unknown item type in '{itemType}'): {resourceName}"); + continue; + } + break; + default: + Log.LogMessage (MessageImportance.Low, $" Not applicable resource (unknown content type '{contentType}'): {resourceName}"); continue; } - var path = Path.Combine (intermediatePath, rpath); + var path = Path.Combine (intermediatePath, itemType, rpath); var file = new FileInfo (path); var item = new TaskItem (path); item.SetMetadata ("LogicalName", rpath); item.SetMetadata ("Optimize", "false"); + item.SetMetadata ("BundledInAssembly", assembly); if (file.Exists && file.LastWriteTimeUtc >= asmWriteTime) { - Log.LogMessage (" Up to date: {0}", rpath); + Log.LogMessage ($" Up to date (contentType: {contentType} resourceType: {resourceType} resourceName: {resourceName}): {path}"); } else { - Log.LogMessage (" Unpacking: {0}", rpath); - - Directory.CreateDirectory (Path.GetDirectoryName (path)); - - var resourceDirectory = peReader.GetSectionData (peReader.PEHeaders.CorHeader!.ResourcesDirectory.RelativeVirtualAddress); - var reader = resourceDirectory.GetReader ((int) manifestResource.Offset, resourceDirectory.Length - (int) manifestResource.Offset); - var length = reader.ReadUInt32 (); - if (length > reader.RemainingBytes) - throw new BadImageFormatException (); -#if NET - using var fs = new FileStream (path, FileMode.Create, FileAccess.Write, FileShare.Read); - unsafe { - var span = new ReadOnlySpan (reader.CurrentPointer, (int) length); - fs.Write (span); - } -#else - var buffer = new byte [4096]; - using var fs = new FileStream (path, FileMode.Create, FileAccess.Write, FileShare.Read, buffer.Length); - var left = (int) length; - while (left > 0) { - var read = Math.Min (left, buffer.Length); - reader.ReadBytes (read, buffer, 0); - fs.Write (buffer, 0, read); - left -= read; - } -#endif + extractor.WriteResourceTo (embedded, path); unpackedResources.Add (item); + Log.LogMessage ($" Unpacked (contentType: {contentType} resourceType: {resourceType} resourceName: {resourceName}): {path}"); } - - rv.Add (item); + rv.Add (new AssemblyContentResource (resourceType, item)); } } catch (Exception e) { Log.LogMessage (MessageImportance.Low, $"Unable to load the resources from the assembly '{assembly}': {e}"); - return new List (); + return new List (); } return rv; } @@ -221,25 +426,6 @@ static string UnmangleResource (string mangled) return unmangled.ToString (); } - public class ManifestResource { - readonly Func callback; - - public ManifestResource (string name, Func streamCallback) - { - callback = streamCallback; - Name = name; - } - - public string Name { - get; private set; - } - - public Stream Open () - { - return callback (); - } - } - public void Cancel () { if (ShouldExecuteRemotely ()) @@ -254,7 +440,12 @@ public bool ShouldCopyToBuildServer (ITaskItem item) return true; } - public bool ShouldCreateOutputFile (ITaskItem item) => UnpackedResources.Contains (item) == true; + public bool ShouldCreateOutputFile (ITaskItem item) + { + // Incremental builds are handled with stamp files in the .targets file, so there's no need to + // create any output files on Windows. + return false; + } public IEnumerable GetAdditionalItemsToBeCopied () => ItemsFiles; diff --git a/msbuild/Xamarin.Shared/Xamarin.Shared.props b/msbuild/Xamarin.Shared/Xamarin.Shared.props index de3df76b9fa5..59af30687bac 100644 --- a/msbuild/Xamarin.Shared/Xamarin.Shared.props +++ b/msbuild/Xamarin.Shared/Xamarin.Shared.props @@ -310,6 +310,12 @@ Copyright (C) 2020 Microsoft. All rights reserved. all <_AppBundleName>$(AssemblyName) + + + true + + + <_BundleOriginalResources Condition="'$(OutputType)' == 'Library' And '$(IsAppExtension)' != 'true' And '$(BundleOriginalResources)' == 'true'">true diff --git a/msbuild/Xamarin.Shared/Xamarin.Shared.targets b/msbuild/Xamarin.Shared/Xamarin.Shared.targets index 31772d6f9e94..aae137b1ac28 100644 --- a/msbuild/Xamarin.Shared/Xamarin.Shared.targets +++ b/msbuild/Xamarin.Shared/Xamarin.Shared.targets @@ -44,6 +44,7 @@ Copyright (C) 2018 Microsoft. All rights reserved. + @@ -120,6 +121,7 @@ Copyright (C) 2018 Microsoft. All rights reserved. False False + False @@ -442,6 +444,7 @@ Copyright (C) 2018 Microsoft. All rights reserved. $(CollectBundleResourcesDependsOn); + _UnpackLibraryResources; _CompileImageAssets; _CompileInterfaceDefinitions; _CompileSceneKitAssets; @@ -470,13 +473,14 @@ Copyright (C) 2018 Microsoft. All rights reserved. @@ -549,6 +553,25 @@ Copyright (C) 2018 Microsoft. All rights reserved. + + + + + + + + @@ -651,7 +675,7 @@ Copyright (C) 2018 Microsoft. All rights reserved. - + - + - + + + + + + + + + + True False + False diff --git a/msbuild/Xamarin.iOS.Tasks.Windows/Xamarin.iOS.Common.After.targets b/msbuild/Xamarin.iOS.Tasks.Windows/Xamarin.iOS.Common.After.targets index 289c51ef5e45..181a7287f5cd 100644 --- a/msbuild/Xamarin.iOS.Tasks.Windows/Xamarin.iOS.Common.After.targets +++ b/msbuild/Xamarin.iOS.Tasks.Windows/Xamarin.iOS.Common.After.targets @@ -201,7 +201,7 @@ Copyright (C) 2011-2013 Xamarin. All rights reserved. - From b12b1062b1df3940abc901ed426858e357876543 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Tue, 30 Jan 2024 12:58:51 +0100 Subject: [PATCH 03/10] [tests] Update a few tests to test bundling original resources. --- tests/dotnet/UnitTests/ProjectTest.cs | 216 +++++++++++++++++--------- 1 file changed, 143 insertions(+), 73 deletions(-) diff --git a/tests/dotnet/UnitTests/ProjectTest.cs b/tests/dotnet/UnitTests/ProjectTest.cs index 2b74832e19db..2da562f5a976 100644 --- a/tests/dotnet/UnitTests/ProjectTest.cs +++ b/tests/dotnet/UnitTests/ProjectTest.cs @@ -789,44 +789,68 @@ public void AppWithResources (ApplePlatform platform, string runtimeIdentifiers) } [Category ("Windows")] - [TestCase (ApplePlatform.iOS)] - public void LibraryWithResourcesOnWindows (ApplePlatform platform) + [TestCase (ApplePlatform.iOS, true)] + [TestCase (ApplePlatform.iOS, false)] + [TestCase (ApplePlatform.iOS, null)] + public void LibraryWithResourcesOnWindows (ApplePlatform platform, bool? bundleOriginalResources) { Configuration.IgnoreIfNotOnWindows (); - LibraryWithResources (platform, anyLibraryResources: false); + // This should all execute locally on Windows when BundleOriginalResources=true + LibraryWithResources (platform, anyLibraryResources: bundleOriginalResources == true, bundleOriginalResources: bundleOriginalResources); } + [Category ("RemoteWindows")] - [TestCase (ApplePlatform.iOS)] - public void LibraryWithResourcesOnRemoteWindows (ApplePlatform platform) + [TestCase (ApplePlatform.iOS, true)] + [TestCase (ApplePlatform.iOS, false)] + public void LibraryWithResourcesOnRemoteWindows (ApplePlatform platform, bool? bundleOriginalResources) { Configuration.IgnoreIfNotOnWindows (); - LibraryWithResources (platform); + // This should all execute locally on Windows when BundleOriginalResources=true, but either should work + LibraryWithResources (platform, bundleOriginalResources); } - [TestCase (ApplePlatform.iOS)] - [TestCase (ApplePlatform.TVOS)] - [TestCase (ApplePlatform.MacCatalyst)] - [TestCase (ApplePlatform.MacOSX)] - public void LibraryWithResources (ApplePlatform platform, bool anyLibraryResources = true) + [TestCase (ApplePlatform.iOS, true)] + [TestCase (ApplePlatform.iOS, false)] + [TestCase (ApplePlatform.iOS, null)] + [TestCase (ApplePlatform.TVOS, true)] + [TestCase (ApplePlatform.TVOS, false)] + [TestCase (ApplePlatform.TVOS, null)] + [TestCase (ApplePlatform.MacCatalyst, true)] + [TestCase (ApplePlatform.MacCatalyst, false)] + [TestCase (ApplePlatform.MacCatalyst, null)] + [TestCase (ApplePlatform.MacOSX, true)] + [TestCase (ApplePlatform.MacOSX, false)] + [TestCase (ApplePlatform.MacOSX, null)] + public void LibraryWithResources (ApplePlatform platform, bool? bundleOriginalResources, bool anyLibraryResources = true) { var project = "LibraryWithResources"; Configuration.IgnoreIfIgnoredPlatform (platform); + var actualBundleOriginalResources = bundleOriginalResources ?? Version.Parse (Configuration.DotNetTfm.Replace ("net", "")).Major >= 10; var project_path = GetProjectPath (project, platform: platform); Clean (project_path); var properties = GetDefaultProperties (); + if (bundleOriginalResources.HasValue) + properties ["BundleOriginalResources"] = bundleOriginalResources.Value ? "true" : "false"; + var rv = DotNet.AssertBuild (project_path, properties); var allTargets = BinLog.GetAllTargets (rv.BinLogPath).Where (v => !v.Skipped).Select (v => v.TargetName); // https://github.com/xamarin/xamarin-macios/issues/15031 - Assert.That (allTargets, Does.Contain ("_CompileAppManifest"), "Did execute '_CompileAppManifest'"); - Assert.That (allTargets, Does.Contain ("_DetectSdkLocations"), "Did execute '_DetectSdkLocations'"); - if (System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform (System.Runtime.InteropServices.OSPlatform.Windows)) - Assert.That (allTargets, Does.Contain ("_SayHello"), "Did execute '_SayHello'"); + if (actualBundleOriginalResources) { + Assert.That (allTargets, Does.Not.Contain ("_CompileAppManifest"), "Didn't execute '_CompileAppManifest'"); + Assert.That (allTargets, Does.Not.Contain ("_DetectSdkLocations"), "Didn't execute '_DetectSdkLocations'"); + Assert.That (allTargets, Does.Not.Contain ("_SayHello"), "Didn't execute '_SayHello'"); + } else { + Assert.That (allTargets, Does.Contain ("_CompileAppManifest"), "Did execute '_CompileAppManifest'"); + Assert.That (allTargets, Does.Contain ("_DetectSdkLocations"), "Did execute '_DetectSdkLocations'"); + if (System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform (System.Runtime.InteropServices.OSPlatform.Windows)) + Assert.That (allTargets, Does.Contain ("_SayHello"), "Did execute '_SayHello'"); + } var lines = BinLog.PrintToLines (rv.BinLogPath); // Find the resulting binding assembly from the build log @@ -844,88 +868,127 @@ public void LibraryWithResources (ApplePlatform platform, bool anyLibraryResourc if (anyLibraryResources) { var platformPrefix = (platform == ApplePlatform.MacOSX) ? "xammac" : "monotouch"; - var expectedList = new List (); - expectedList.Add ($"__{platformPrefix}_content_A.ttc"); - expectedList.Add ($"__{platformPrefix}_content_Archer__Attack.atlasc_sArcher__Attack.plist"); - expectedList.Add ($"__{platformPrefix}_content_art.scnassets_sscene.scn"); - expectedList.Add ($"__{platformPrefix}_content_art.scnassets_stexture.png"); - expectedList.Add ($"__{platformPrefix}_content_Assets.car"); - expectedList.Add ($"__{platformPrefix}_content_B.otf"); - expectedList.Add ($"__{platformPrefix}_content_C.ttf"); - expectedList.Add ($"__{platformPrefix}_content_DirWithResources_slinkedArt.scnassets_sscene.scn"); - expectedList.Add ($"__{platformPrefix}_content_DirWithResources_slinkedArt.scnassets_stexture.png"); - expectedList.Add ($"__{platformPrefix}_content_scene.dae"); - switch (platform) { - case ApplePlatform.iOS: - expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sBYZ-38-t0r-view-8bC-Xf-vdC.nib"); - expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sInfo.plist"); - expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sUIViewController-BYZ-38-t0r.nib"); - break; - case ApplePlatform.TVOS: - expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sBYZ-38-t0r-view-8bC-Xf-vdC.nib"); - expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sInfo.plist"); - expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sUIViewController-BYZ-38-t0r.nib"); - break; - case ApplePlatform.MacCatalyst: - expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_s1-view-2.nib"); - expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sInfo.plist"); - expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sUIViewController-1.nib"); - break; - case ApplePlatform.MacOSX: - expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sInfo.plist"); - expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sMainMenu.nib"); - expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sNSWindowController-B8D-0N-5wS.nib"); - expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sXfG-lQ-9wD-view-m2S-Jp-Qdl.nib"); - break; + if (actualBundleOriginalResources) { + expectedResources = new string [] { + $"__{platformPrefix}_content_A.ttc", + $"__{platformPrefix}_content_B.otf", + $"__{platformPrefix}_content_C.ttf", + $"__{platformPrefix}_item_AtlasTexture_Archer__Attack.atlas_sarcher__attack__0001.png", + $"__{platformPrefix}_item_AtlasTexture_Archer__Attack.atlas_sarcher__attack__0002.png", + $"__{platformPrefix}_item_AtlasTexture_Archer__Attack.atlas_sarcher__attack__0003.png", + $"__{platformPrefix}_item_AtlasTexture_Archer__Attack.atlas_sarcher__attack__0004.png", + $"__{platformPrefix}_item_AtlasTexture_Archer__Attack.atlas_sarcher__attack__0005.png", + $"__{platformPrefix}_item_AtlasTexture_Archer__Attack.atlas_sarcher__attack__0006.png", + $"__{platformPrefix}_item_AtlasTexture_Archer__Attack.atlas_sarcher__attack__0007.png", + $"__{platformPrefix}_item_AtlasTexture_Archer__Attack.atlas_sarcher__attack__0008.png", + $"__{platformPrefix}_item_AtlasTexture_Archer__Attack.atlas_sarcher__attack__0009.png", + $"__{platformPrefix}_item_AtlasTexture_Archer__Attack.atlas_sarcher__attack__0010.png", + $"__{platformPrefix}_item_BundleResource_A.ttc", + $"__{platformPrefix}_item_BundleResource_B.otf", + $"__{platformPrefix}_item_BundleResource_C.ttf", + $"__{platformPrefix}_item_Collada_scene.dae", + $"__{platformPrefix}_item_CoreMLModel_SqueezeNet.mlmodel", + $"__{platformPrefix}_item_ImageAsset_Images.xcassets_sContents.json", + $"__{platformPrefix}_item_ImageAsset_Images.xcassets_sImage.imageset_sContents.json", + $"__{platformPrefix}_item_ImageAsset_Images.xcassets_sImage.imageset_sIcon16.png", + $"__{platformPrefix}_item_ImageAsset_Images.xcassets_sImage.imageset_sIcon32.png", + $"__{platformPrefix}_item_ImageAsset_Images.xcassets_sImage.imageset_sIcon64.png", + $"__{platformPrefix}_item_InterfaceDefinition_Main.storyboard", + $"__{platformPrefix}_item_PartialAppManifest_shared.plist", + $"__{platformPrefix}_item_SceneKitAsset_art.scnassets_sscene.scn", + $"__{platformPrefix}_item_SceneKitAsset_art.scnassets_stexture.png", + $"__{platformPrefix}_item_SceneKitAsset_DirWithResources_slinkedArt.scnassets_sscene.scn", + $"__{platformPrefix}_item_SceneKitAsset_DirWithResources_slinkedArt.scnassets_stexture.png", + }; + } else { + var expectedList = new List (); + expectedList.Add ($"__{platformPrefix}_content_A.ttc"); + expectedList.Add ($"__{platformPrefix}_content_Archer__Attack.atlasc_sArcher__Attack.plist"); + expectedList.Add ($"__{platformPrefix}_content_art.scnassets_sscene.scn"); + expectedList.Add ($"__{platformPrefix}_content_art.scnassets_stexture.png"); + expectedList.Add ($"__{platformPrefix}_content_Assets.car"); + expectedList.Add ($"__{platformPrefix}_content_B.otf"); + expectedList.Add ($"__{platformPrefix}_content_C.ttf"); + expectedList.Add ($"__{platformPrefix}_content_DirWithResources_slinkedArt.scnassets_sscene.scn"); + expectedList.Add ($"__{platformPrefix}_content_DirWithResources_slinkedArt.scnassets_stexture.png"); + expectedList.Add ($"__{platformPrefix}_content_scene.dae"); + switch (platform) { + case ApplePlatform.iOS: + expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sBYZ-38-t0r-view-8bC-Xf-vdC.nib"); + expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sInfo.plist"); + expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sUIViewController-BYZ-38-t0r.nib"); + break; + case ApplePlatform.TVOS: + expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sBYZ-38-t0r-view-8bC-Xf-vdC.nib"); + expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sInfo.plist"); + expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sUIViewController-BYZ-38-t0r.nib"); + break; + case ApplePlatform.MacCatalyst: + expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_s1-view-2.nib"); + expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sInfo.plist"); + expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sUIViewController-1.nib"); + break; + case ApplePlatform.MacOSX: + expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sInfo.plist"); + expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sMainMenu.nib"); + expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sNSWindowController-B8D-0N-5wS.nib"); + expectedList.Add ($"__{platformPrefix}_content_Main.storyboardc_sXfG-lQ-9wD-view-m2S-Jp-Qdl.nib"); + break; + } + expectedList.Add ($"__{platformPrefix}_content_SqueezeNet.mlmodelc_sanalytics_scoremldata.bin"); + expectedList.Add ($"__{platformPrefix}_content_SqueezeNet.mlmodelc_scoremldata.bin"); + expectedList.Add ($"__{platformPrefix}_content_SqueezeNet.mlmodelc_smetadata.json"); + expectedList.Add ($"__{platformPrefix}_content_SqueezeNet.mlmodelc_smodel.espresso.net"); + expectedList.Add ($"__{platformPrefix}_content_SqueezeNet.mlmodelc_smodel.espresso.shape"); + expectedList.Add ($"__{platformPrefix}_content_SqueezeNet.mlmodelc_smodel.espresso.weights"); + expectedList.Add ($"__{platformPrefix}_content_SqueezeNet.mlmodelc_smodel_scoremldata.bin"); + expectedList.Add ($"__{platformPrefix}_content_SqueezeNet.mlmodelc_sneural__network__optionals_scoremldata.bin"); + expectedResources = expectedList.ToArray (); } - expectedList.Add ($"__{platformPrefix}_content_SqueezeNet.mlmodelc_sanalytics_scoremldata.bin"); - expectedList.Add ($"__{platformPrefix}_content_SqueezeNet.mlmodelc_scoremldata.bin"); - expectedList.Add ($"__{platformPrefix}_content_SqueezeNet.mlmodelc_smetadata.json"); - expectedList.Add ($"__{platformPrefix}_content_SqueezeNet.mlmodelc_smodel.espresso.net"); - expectedList.Add ($"__{platformPrefix}_content_SqueezeNet.mlmodelc_smodel.espresso.shape"); - expectedList.Add ($"__{platformPrefix}_content_SqueezeNet.mlmodelc_smodel.espresso.weights"); - expectedList.Add ($"__{platformPrefix}_content_SqueezeNet.mlmodelc_smodel_scoremldata.bin"); - expectedList.Add ($"__{platformPrefix}_content_SqueezeNet.mlmodelc_sneural__network__optionals_scoremldata.bin"); - expectedResources = expectedList.ToArray (); } else { expectedResources = new string [0]; } CollectionAssert.AreEquivalent (expectedResources, actualResources, "Resources"); } - [TestCase (ApplePlatform.iOS, "ios-arm64")] - [TestCase (ApplePlatform.TVOS, "tvossimulator-arm64")] - [TestCase (ApplePlatform.MacCatalyst, "maccatalyst-x64")] - [TestCase (ApplePlatform.MacCatalyst, "maccatalyst-arm64;maccatalyst-x64")] - [TestCase (ApplePlatform.MacOSX, "osx-x64")] - [TestCase (ApplePlatform.MacOSX, "osx-arm64;osx-x64")] - public void AppWithLibraryWithResourcesReference (ApplePlatform platform, string runtimeIdentifiers) + [TestCase (ApplePlatform.iOS, "ios-arm64", false)] + [TestCase (ApplePlatform.iOS, "ios-arm64", true)] + [TestCase (ApplePlatform.TVOS, "tvossimulator-arm64", false)] + [TestCase (ApplePlatform.TVOS, "tvossimulator-arm64", true)] + [TestCase (ApplePlatform.MacCatalyst, "maccatalyst-x64", false)] + [TestCase (ApplePlatform.MacCatalyst, "maccatalyst-arm64;maccatalyst-x64", true)] + [TestCase (ApplePlatform.MacOSX, "osx-x64", true)] + [TestCase (ApplePlatform.MacOSX, "osx-arm64;osx-x64", false)] + public void AppWithLibraryWithResourcesReference (ApplePlatform platform, string runtimeIdentifiers, bool bundleOriginalResources) { - AppWithLibraryWithResourcesReferenceImpl (platform, runtimeIdentifiers, false, false); + AppWithLibraryWithResourcesReferenceImpl (platform, runtimeIdentifiers, bundleOriginalResources, false, false); } + [Category ("RemoteWindows")] - [TestCase (ApplePlatform.iOS, "ios-arm64")] - public void AppWithLibraryWithResourcesReferenceOnRemoteWindows (ApplePlatform platform, string runtimeIdentifiers) + [TestCase (ApplePlatform.iOS, "ios-arm64", false)] + [TestCase (ApplePlatform.iOS, "ios-arm64", true)] + public void AppWithLibraryWithResourcesReferenceOnRemoteWindows (ApplePlatform platform, string runtimeIdentifiers, bool bundleOriginalResources) { Configuration.IgnoreIfNotOnWindows (); - AppWithLibraryWithResourcesReferenceImpl (platform, runtimeIdentifiers, true, false); + AppWithLibraryWithResourcesReferenceImpl (platform, runtimeIdentifiers, bundleOriginalResources, true, false); } [Category ("Windows")] - [TestCase (ApplePlatform.iOS, "ios-arm64")] - public void AppWithLibraryWithResourcesReferenceWithHotRestart (ApplePlatform platform, string runtimeIdentifiers) + [TestCase (ApplePlatform.iOS, "ios-arm64", false)] + [TestCase (ApplePlatform.iOS, "ios-arm64", true)] + public void AppWithLibraryWithResourcesReferenceWithHotRestart (ApplePlatform platform, string runtimeIdentifiers, bool bundleOriginalResources) { Configuration.IgnoreIfNotOnWindows (); - AppWithLibraryWithResourcesReferenceImpl (platform, runtimeIdentifiers, false, isUsingHotRestart: true); + AppWithLibraryWithResourcesReferenceImpl (platform, runtimeIdentifiers, bundleOriginalResources, false, isUsingHotRestart: true); } - void AppWithLibraryWithResourcesReferenceImpl (ApplePlatform platform, string runtimeIdentifiers, bool remoteWindows, bool isUsingHotRestart) + void AppWithLibraryWithResourcesReferenceImpl (ApplePlatform platform, string runtimeIdentifiers, bool bundleOriginalResources, bool remoteWindows, bool isUsingHotRestart) { var project = "AppWithLibraryWithResourcesReference"; - var config = "Debug"; + var config = bundleOriginalResources ? "DebugOriginal" : "DebugCompiled"; Configuration.IgnoreIfIgnoredPlatform (platform); Configuration.AssertRuntimeIdentifiersAvailable (platform, runtimeIdentifiers); @@ -946,6 +1009,7 @@ void AppWithLibraryWithResourcesReferenceImpl (ApplePlatform platform, string ru var properties = GetDefaultProperties (runtimeIdentifiers, extraProperties); properties ["Configuration"] = config; + properties ["BundleOriginalResources"] = bundleOriginalResources ? "true" : "false"; if (remoteWindows) { // Copy the app bundle to Windows so that we can inspect the results. properties ["CopyAppBundleToWindows"] = "true"; @@ -1005,6 +1069,12 @@ void AppWithLibraryWithResourcesReferenceImpl (ApplePlatform platform, string ru AssertExistsOrUsingHotRestart (Path.Combine (mlModel, "model", "coremldata.bin"), "CoreMLModel/model/coremldata.bin"); AssertExistsOrUsingHotRestart (Path.Combine (mlModel, "neural_network_optionals"), "CoreMLModel/neural_network_optionals"); AssertExistsOrUsingHotRestart (Path.Combine (mlModel, "neural_network_optionals", "coremldata.bin"), "CoreMLModel/neural_network_optionals/coremldata.bin"); + + if (bundleOriginalResources) { + var infoPlist = appBundleInfo.GetFile (GetInfoPListPath (platform, "")); + var appManifest = PDictionary.FromByteArray (infoPlist, out var _)!; + Assert.AreEqual ("Here I am", appManifest.GetString ("LibraryWithResources").Value, "Partial plist entry"); + } }); void AssertExistsOrUsingHotRestart (string path, string message) From f52fd194529da0907fc96ba119d53cd07a4ec646 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Mon, 29 Apr 2024 14:44:04 +0200 Subject: [PATCH 04/10] [msbuild] Fix LogicalName for SceneKitAsset items. The SceneKitAsset items contains all the resources within a .scnasset directory. However, we need to pass the .scnasset directory to the native tooling, so we compute the path to the .scnasset directory, and create a new item for it. This new item must have the correct LogicalName set, which is accomplished by: * Not deleting the LogicalName from the original item before using it to compute the new item's LogicalName. * Compute the correct LogicalName for the .scnasset, and actually set it. This fixes an issue where we'd be unable to detect previously processed SceneKitAsset items in universal builds, because the files wouldn't located where they should be on disk. --- .../Tasks/CompileSceneKitAssets.cs | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/CompileSceneKitAssets.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/CompileSceneKitAssets.cs index 7b4adf3adb9f..46be6f6f2d1c 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/CompileSceneKitAssets.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/CompileSceneKitAssets.cs @@ -118,6 +118,14 @@ Task CopySceneKitAssets (string scnassets, string output, string intermediate) return ExecuteAsync (GetFullPathToTool (), args, sdkDevPath: SdkDevPath, environment: environment, showErrorIfFailure: true); } + static bool TryGetScnAssetsPath (string file, out string scnassets) + { + scnassets = file; + while (scnassets.Length > 0 && Path.GetExtension (scnassets).ToLowerInvariant () != ".scnassets") + scnassets = Path.GetDirectoryName (scnassets); + return scnassets.Length > 0; + } + public override bool Execute () { if (ShouldExecuteRemotely ()) { @@ -140,15 +148,9 @@ public override bool Execute () continue; // get the .scnassets directory path - var scnassets = Path.GetDirectoryName (asset.ItemSpec); - while (scnassets.Length > 0 && Path.GetExtension (scnassets).ToLowerInvariant () != ".scnassets") - scnassets = Path.GetDirectoryName (scnassets); - - if (scnassets.Length == 0) + if (!TryGetScnAssetsPath (asset.ItemSpec, out var scnassets)) continue; - asset.RemoveMetadata ("LogicalName"); - var bundleName = BundleResource.GetLogicalName (this, asset); var output = new TaskItem (Path.Combine (intermediate, bundleName)); @@ -159,6 +161,13 @@ public override bool Execute () // .. but we really want it to be for @scnassets, so set ItemSpec accordingly scnassetsItem.ItemSpec = scnassets; + // .. and set LogicalName, the original one is for @asset + if (!TryGetScnAssetsPath (bundleName, out var logicalScnAssetsPath)) { + Log.LogError (null, null, null, asset.ItemSpec, $"Unable to compute the path of the *.scnassets path from the item's LogicalName '{bundleName}'"); + continue; + } + scnassetsItem.SetMetadata ("LogicalName", logicalScnAssetsPath); + // .. and remove the @OriginalItemSpec which is for @asset scnassetsItem.RemoveMetadata ("OriginalItemSpec"); From cfc46442f022ba1875ce70c0ab98eef532981c12 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 5 Jun 2024 16:46:04 +0200 Subject: [PATCH 05/10] [msbuild] Don't execute the _CreateEmbeddedResources and _PackLibraryResources targets remotely. This can happen entirely on Windows. --- .../Tasks/CreateEmbeddedResources.cs | 11 ------ .../Tasks/PackLibraryResources.cs | 35 ------------------- .../Xamarin.Shared.ObjCBinding.targets | 1 - msbuild/Xamarin.Shared/Xamarin.Shared.targets | 2 -- 4 files changed, 49 deletions(-) diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/CreateEmbeddedResources.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/CreateEmbeddedResources.cs index e550c47af9b4..c1da950e5fb7 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/CreateEmbeddedResources.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/CreateEmbeddedResources.cs @@ -19,17 +19,6 @@ public class CreateEmbeddedResources : XamarinTask { public override bool Execute () { - if (ShouldExecuteRemotely ()) { - foreach (var bundleResource in this.BundleResources) { - var logicalName = bundleResource.GetMetadata ("LogicalName"); - - if (!string.IsNullOrEmpty (logicalName)) { - logicalName = logicalName.Replace ("\\", "/"); - bundleResource.SetMetadata ("LogicalName", logicalName); - } - } - } - EmbeddedResources = new ITaskItem [BundleResources.Length]; for (int i = 0; i < BundleResources.Length; i++) { diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/PackLibraryResources.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/PackLibraryResources.cs index afb93f312004..ec38004711c9 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/PackLibraryResources.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/PackLibraryResources.cs @@ -46,43 +46,8 @@ public static string EscapeMangledResource (string name) return mangled.ToString (); } - bool ExecuteRemotely () - { - // Fix LogicalName path for the Mac - foreach (var resource in BundleResourcesWithLogicalNames) { - var logicalName = resource.GetMetadata ("LogicalName"); - - if (!string.IsNullOrEmpty (logicalName)) { - resource.SetMetadata ("LogicalName", logicalName.Replace ("\\", "/")); - } - } - - var runner = new TaskRunner (SessionId, BuildEngine4); - - try { - var result = runner.RunAsync (this).Result; - - if (result && EmbeddedResources is not null) { - // We must get the "real" file that will be embedded in the - // compiled assembly in Windows - foreach (var embeddedResource in EmbeddedResources.Where (x => runner.ShouldCopyItemAsync (task: this, item: x).Result)) { - runner.GetFileAsync (this, embeddedResource.ItemSpec).Wait (); - } - } - - return result; - } catch (Exception ex) { - Log.LogErrorFromException (ex); - - return false; - } - } - public override bool Execute () { - if (ShouldExecuteRemotely ()) - return ExecuteRemotely (); - var results = new List (); foreach (var item in BundleResourcesWithLogicalNames) { diff --git a/msbuild/Xamarin.Shared/Xamarin.Shared.ObjCBinding.targets b/msbuild/Xamarin.Shared/Xamarin.Shared.ObjCBinding.targets index 0859ac813c67..b3dac7c482a8 100644 --- a/msbuild/Xamarin.Shared/Xamarin.Shared.ObjCBinding.targets +++ b/msbuild/Xamarin.Shared/Xamarin.Shared.ObjCBinding.targets @@ -103,7 +103,6 @@ Copyright (C) 2020 Microsoft. All rights reserved. diff --git a/msbuild/Xamarin.Shared/Xamarin.Shared.targets b/msbuild/Xamarin.Shared/Xamarin.Shared.targets index aae137b1ac28..dc5740fddf08 100644 --- a/msbuild/Xamarin.Shared/Xamarin.Shared.targets +++ b/msbuild/Xamarin.Shared/Xamarin.Shared.targets @@ -2087,8 +2087,6 @@ Copyright (C) 2018 Microsoft. All rights reserved. Date: Wed, 4 Dec 2024 10:06:26 +0100 Subject: [PATCH 06/10] Localize --- msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx | 6 +++++- msbuild/Xamarin.MacDev.Tasks/Tasks/CompileSceneKitAssets.cs | 2 +- .../Xamarin.MacDev.Tasks/Tasks/UnpackLibraryResources.cs | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx b/msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx index 21c4a18f256d..3b2a934e4bdb 100644 --- a/msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx +++ b/msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx @@ -1655,7 +1655,11 @@ - + + Unknown resource type: {1}. + + + Unknown resource type: {1}. diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/CompileSceneKitAssets.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/CompileSceneKitAssets.cs index 46be6f6f2d1c..a16483592ad6 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/CompileSceneKitAssets.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/CompileSceneKitAssets.cs @@ -163,7 +163,7 @@ public override bool Execute () // .. and set LogicalName, the original one is for @asset if (!TryGetScnAssetsPath (bundleName, out var logicalScnAssetsPath)) { - Log.LogError (null, null, null, asset.ItemSpec, $"Unable to compute the path of the *.scnassets path from the item's LogicalName '{bundleName}'"); + Log.LogError (null, null, null, asset.ItemSpec, MSBStrings.E7136 /* Unable to compute the path of the *.scnassets path from the item's LogicalName '{0}'. */ , bundleName); continue; } scnassetsItem.SetMetadata ("LogicalName", logicalScnAssetsPath); diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/UnpackLibraryResources.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/UnpackLibraryResources.cs index 0bb15a4cf5f8..2d3f426b53ca 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/UnpackLibraryResources.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/UnpackLibraryResources.cs @@ -147,7 +147,7 @@ public override bool Execute () sceneKitAssets.Add (item); break; default: - Log.LogError (MSBStrings.E7134 /* Unknown resource type: {1}. */, resourceType); + Log.LogError (MSBStrings.E7135 /* Unknown resource type: {1}. */, resourceType); break; } } From b215385cdf86c4ac1069fa686c102b9c0dfc42f5 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 4 Dec 2024 10:10:00 +0100 Subject: [PATCH 07/10] Remove unnecessary remoting code. --- .../Tasks/PackLibraryResources.cs | 11 +----- .../Tasks/UnpackLibraryResources.cs | 38 +------------------ 2 files changed, 2 insertions(+), 47 deletions(-) diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/PackLibraryResources.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/PackLibraryResources.cs index ec38004711c9..2c8cca4505ba 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/PackLibraryResources.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/PackLibraryResources.cs @@ -7,10 +7,9 @@ using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using Xamarin.Localization.MSBuild; -using Xamarin.Messaging.Build.Client; namespace Xamarin.MacDev.Tasks { - public class PackLibraryResources : XamarinTask, ITaskCallback, ICancelableTask { + public class PackLibraryResources : XamarinTask, ICancelableTask { #region Inputs [Required] @@ -93,14 +92,6 @@ bool TryGetMangledLogicalName (ITaskItem item, string itemName, [NotNullWhen (tr public void Cancel () { - if (ShouldExecuteRemotely ()) - BuildConnection.CancelAsync (BuildEngine4).Wait (); } - - public bool ShouldCopyToBuildServer (ITaskItem item) => false; - - public bool ShouldCreateOutputFile (ITaskItem item) => false; - - public IEnumerable GetAdditionalItemsToBeCopied () => Enumerable.Empty (); } } diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/UnpackLibraryResources.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/UnpackLibraryResources.cs index 2d3f426b53ca..d0081325f780 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/UnpackLibraryResources.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/UnpackLibraryResources.cs @@ -12,12 +12,11 @@ using Microsoft.Build.Utilities; using Xamarin.Localization.MSBuild; -using Xamarin.Messaging.Build.Client; #nullable enable namespace Xamarin.MacDev.Tasks { - public class UnpackLibraryResources : XamarinTask, ITaskCallback, ICancelableTask { + public class UnpackLibraryResources : XamarinTask, ICancelableTask { List unpackedResources = new List (); #region Inputs @@ -84,22 +83,6 @@ enum ResourceType { public override bool Execute () { - if (ShouldExecuteRemotely ()) { - var result = new TaskRunner (SessionId, BuildEngine4).RunAsync (this).Result; - - if (result && BundleResourcesWithLogicalNames is not null) { - // Fix LogicalName path for Windows - foreach (var resource in BundleResourcesWithLogicalNames) { - var logicalName = resource.GetMetadata ("LogicalName"); - - if (!string.IsNullOrEmpty (logicalName)) { - resource.SetMetadata ("LogicalName", logicalName.Replace ("/", "\\")); - } - } - } - return result; - } - var bundleResources = new List (); var atlasTextures = new List (); var colladaAssets = new List (); @@ -428,26 +411,7 @@ static string UnmangleResource (string mangled) public void Cancel () { - if (ShouldExecuteRemotely ()) - BuildConnection.CancelAsync (BuildEngine4).Wait (); - } - - public bool ShouldCopyToBuildServer (ITaskItem item) - { - if (item.IsFrameworkItem ()) - return false; - - return true; - } - - public bool ShouldCreateOutputFile (ITaskItem item) - { - // Incremental builds are handled with stamp files in the .targets file, so there's no need to - // create any output files on Windows. - return false; } - public IEnumerable GetAdditionalItemsToBeCopied () => ItemsFiles; - } } From 6de0adf751d8b0e5d3153fee1fc6d7c37f702f4f Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 4 Dec 2024 13:15:48 +0100 Subject: [PATCH 08/10] Fix InvalidSupportedOSPlatformVersion test. The execution of targets has changed a little bit: now we build referenced projects earlier. For this particular test, it meant building both the NUnitLite and Touch.Unit projects first, and they both raised the expected error. However, the test only expects a single error, so skip building NUnitLite and Touch.Unit, in which case the test will only find the single expected error. --- tests/dotnet/UnitTests/ProjectTest.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/dotnet/UnitTests/ProjectTest.cs b/tests/dotnet/UnitTests/ProjectTest.cs index 2da562f5a976..e451c15f76fb 100644 --- a/tests/dotnet/UnitTests/ProjectTest.cs +++ b/tests/dotnet/UnitTests/ProjectTest.cs @@ -3172,6 +3172,8 @@ public void InvalidSupportedOSPlatformVersion (ApplePlatform platform, string ru Clean (project_path); var properties = GetDefaultProperties (runtimeIdentifiers); properties ["SupportedOSPlatformVersion"] = version; + properties ["ExcludeTouchUnitReference"] = "true"; + properties ["ExcludeNUnitLiteReference"] = "true"; var rv = DotNet.AssertBuildFailure (project_path, properties); var errors = BinLog.GetBuildLogErrors (rv.BinLogPath).ToArray (); AssertErrorMessages (errors, $"The SupportedOSPlatformVersion value '{version}' in the project file is lower than the minimum value '{minVersion}'."); From 2009ce57848d4e577a3257645ce169795796e31f Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 4 Dec 2024 18:29:12 +0100 Subject: [PATCH 09/10] [msbuild] Don't collect resources from referenced projects if those projects are bundling original resources. --- .../Xamarin.iOS.Tasks.Windows/Xamarin.iOS.Common.After.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msbuild/Xamarin.iOS.Tasks.Windows/Xamarin.iOS.Common.After.targets b/msbuild/Xamarin.iOS.Tasks.Windows/Xamarin.iOS.Common.After.targets index 181a7287f5cd..6e6823104227 100644 --- a/msbuild/Xamarin.iOS.Tasks.Windows/Xamarin.iOS.Common.After.targets +++ b/msbuild/Xamarin.iOS.Tasks.Windows/Xamarin.iOS.Common.After.targets @@ -36,7 +36,7 @@ Copyright (C) 2011-2013 Xamarin. All rights reserved. - + <_BundleResourceWithLogicalName> $(BuildSessionId) From cd663b456af2717de16fb009c2396ad315e8b200 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 4 Dec 2024 20:49:04 +0100 Subject: [PATCH 10/10] Set on target instead. --- .../Xamarin.iOS.Common.After.targets | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msbuild/Xamarin.iOS.Tasks.Windows/Xamarin.iOS.Common.After.targets b/msbuild/Xamarin.iOS.Tasks.Windows/Xamarin.iOS.Common.After.targets index 6e6823104227..a0b94b1c72bf 100644 --- a/msbuild/Xamarin.iOS.Tasks.Windows/Xamarin.iOS.Common.After.targets +++ b/msbuild/Xamarin.iOS.Tasks.Windows/Xamarin.iOS.Common.After.targets @@ -35,8 +35,8 @@ Copyright (C) 2011-2013 Xamarin. All rights reserved. - - + + <_BundleResourceWithLogicalName> $(BuildSessionId)