From 1a9a1ce73d66f45b4f09c6145d03dd2692819566 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Thu, 19 Jan 2023 22:55:40 +0000 Subject: [PATCH 1/3] [wasm] ManagedToNativeGenerator: Skip unmanaged dlls .. instead crashing with an exception like: ``` src/mono/wasm/build/WasmApp.Native.targets(296,5): error MSB4018: (NETCORE_ENGINEERING_TELEMETRY=Build) The "ManagedToNativeGenerator" task failed unexpectedly. System.BadImageFormatException: This PE image is not a managed executable. at System.Reflection.MetadataLoadContext.LoadFromStreamCore(Stream peStream) at System.Reflection.MetadataLoadContext.LoadFromAssemblyPath(String assemblyPath) at PInvokeTableGenerator.Generate(String[] pinvokeModules, String[] assemblies, String outputPath) in /_/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs:line 42 at ManagedToNativeGenerator.ExecuteInternal() in /_/src/tasks/WasmAppBuilder/ManagedToNativeGenerator.cs:line 68 at ManagedToNativeGenerator.Execute() in /_/src/tasks/WasmAppBuilder/ManagedToNativeGenerator.cs:line 53 at Microsoft.Build.BackEnd.TaskExecutionHost.Microsoft.Build.BackEnd.ITaskExecutionHost.Execute() at Microsoft.Build.BackEnd.TaskBuilder.ExecuteInstantiatedTask(ITaskExecutionHost taskExecutionHost, TaskLoggingContext taskLoggingContext, TaskHost taskHost, ItemBucket bucket, TaskExecutionMode howToExecuteTask) ``` The wasm targets currently are not able to differentiate the managed assemblies from the unmanaged ones. so it needs to be handled here. --- .../WasmAppBuilder/IcallTableGenerator.cs | 8 ++- .../ManagedToNativeGenerator.cs | 50 +++++++++++++++++-- .../WasmAppBuilder/PInvokeTableGenerator.cs | 8 ++- 3 files changed, 59 insertions(+), 7 deletions(-) diff --git a/src/tasks/WasmAppBuilder/IcallTableGenerator.cs b/src/tasks/WasmAppBuilder/IcallTableGenerator.cs index 9416c71760d0fd..f60f84de156727 100644 --- a/src/tasks/WasmAppBuilder/IcallTableGenerator.cs +++ b/src/tasks/WasmAppBuilder/IcallTableGenerator.cs @@ -44,9 +44,13 @@ public IEnumerable Generate(string? runtimeIcallTableFile, string[] asse var resolver = new PathAssemblyResolver(assemblies); using var mlc = new MetadataLoadContext(resolver, "System.Private.CoreLib"); - foreach (var aname in assemblies) + foreach (var asmPath in assemblies) { - var a = mlc.LoadFromAssemblyPath(aname); + if (!File.Exists(asmPath)) + throw new LogAsErrorException($"Cannot find assembly {asmPath}"); + + Log.LogMessage(MessageImportance.Low, $"Loading {asmPath} to scan for icalls"); + var a = mlc.LoadFromAssemblyPath(asmPath); foreach (var type in a.GetTypes()) ProcessType(type); } diff --git a/src/tasks/WasmAppBuilder/ManagedToNativeGenerator.cs b/src/tasks/WasmAppBuilder/ManagedToNativeGenerator.cs index 1dff74a9dcc1c1..37d7da99d6bf48 100644 --- a/src/tasks/WasmAppBuilder/ManagedToNativeGenerator.cs +++ b/src/tasks/WasmAppBuilder/ManagedToNativeGenerator.cs @@ -1,9 +1,12 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.IO; using System.Linq; +using System.Reflection.PortableExecutable; using System.Text; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; @@ -11,7 +14,7 @@ public class ManagedToNativeGenerator : Task { [Required] - public string[]? Assemblies { get; set; } + public string[] Assemblies { get; set; } = Array.Empty(); public string? RuntimeIcallTableFile { get; set; } @@ -62,12 +65,14 @@ public override bool Execute() private void ExecuteInternal() { + string[] managedAssemblies = FilterOutUnmanagedBinaries(Assemblies); + var pinvoke = new PInvokeTableGenerator(FixupSymbolName, Log); var icall = new IcallTableGenerator(FixupSymbolName, Log); IEnumerable cookies = Enumerable.Concat( - pinvoke.Generate(PInvokeModules, Assemblies!, PInvokeOutputPath!), - icall.Generate(RuntimeIcallTableFile, Assemblies!, IcallOutputPath) + pinvoke.Generate(PInvokeModules, managedAssemblies, PInvokeOutputPath!), + icall.Generate(RuntimeIcallTableFile, managedAssemblies, IcallOutputPath) ); var m2n = new InterpToNativeGenerator(Log); @@ -110,4 +115,43 @@ public string FixupSymbolName(string name) _symbolNameFixups[name] = fixedName; return fixedName; } + + private string[] FilterOutUnmanagedBinaries(string[] assemblies) + { + List managedAssemblies = new(assemblies.Length); + foreach (string asmPath in Assemblies) + { + if (!File.Exists(asmPath)) + throw new LogAsErrorException($"Cannot find assembly {asmPath}"); + + try + { + if (!IsManagedAssembly(asmPath)) + { + Log.LogMessage(MessageImportance.Low, $"Skipping unmanaged {asmPath}."); + continue; + } + } + catch (Exception ex) + { + Log.LogMessage(MessageImportance.Low, $"Failed to read assembly {asmPath}: {ex}"); + throw new LogAsErrorException($"Failed to read assembly {asmPath}: {ex.Message}"); + } + + managedAssemblies.Add(asmPath); + } + + return managedAssemblies.ToArray(); + } + + private static bool IsManagedAssembly(string filePath) + { + if (!File.Exists(filePath)) + return false; + + using FileStream fileStream = File.OpenRead(filePath); + using PEReader reader = new(fileStream, PEStreamOptions.Default); + return reader.HasMetadata; + } + } diff --git a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs index ab4eac397bc9f6..389c7f7bd5b3a8 100644 --- a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs +++ b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs @@ -37,9 +37,13 @@ public IEnumerable Generate(string[] pinvokeModules, string[] assemblies var resolver = new PathAssemblyResolver(assemblies); using var mlc = new MetadataLoadContext(resolver, "System.Private.CoreLib"); - foreach (var aname in assemblies) + foreach (var asmPath in assemblies) { - var a = mlc.LoadFromAssemblyPath(aname); + if (!File.Exists(asmPath)) + throw new LogAsErrorException($"Cannot find assembly {asmPath}"); + + Log.LogMessage(MessageImportance.Low, $"Loading {asmPath} to scan for pinvokes"); + var a = mlc.LoadFromAssemblyPath(asmPath); foreach (var type in a.GetTypes()) CollectPInvokes(pinvokes, callbacks, signatures, type); } From de9cc8ab8d742891e7126821d102d18dc08c19d9 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 20 Jan 2023 01:03:12 +0000 Subject: [PATCH 2/3] [wasm] WasmAppHost: Add support for host arguments --- src/mono/wasm/host/CommonConfiguration.cs | 8 +++++++- src/mono/wasm/host/JSEngineHost.cs | 5 ++--- src/mono/wasm/host/RuntimeConfigJson.cs | 2 ++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/mono/wasm/host/CommonConfiguration.cs b/src/mono/wasm/host/CommonConfiguration.cs index eb0a0af997d82d..7d854cc47141e2 100644 --- a/src/mono/wasm/host/CommonConfiguration.cs +++ b/src/mono/wasm/host/CommonConfiguration.cs @@ -22,6 +22,7 @@ internal sealed class CommonConfiguration public WasmHost Host { get; init; } public HostConfig HostConfig { get; init; } public WasmHostProperties HostProperties { get; init; } + public IEnumerable HostArguments { get; init; } private string? hostArg; private string? _runtimeConfigPath; @@ -30,11 +31,13 @@ internal sealed class CommonConfiguration private CommonConfiguration(string[] args) { + List hostArgsList = new(); var options = new OptionSet { { "debug|d", "Start debug server", _ => Debugging = true }, { "host|h=", "Host config name", v => hostArg = v }, - { "runtime-config|r=", "runtimeconfig.json path for the app", v => _runtimeConfigPath = v } + { "runtime-config|r=", "runtimeconfig.json path for the app", v => _runtimeConfigPath = v }, + { "extra-host-arg=", "Extra argument to be passed to the host", hostArgsList.Add }, }; RemainingArgs = options.Parse(args); @@ -95,6 +98,9 @@ private CommonConfiguration(string[] args) if (!Enum.TryParse(HostConfig.HostString, ignoreCase: true, out WasmHost wasmHost)) throw new CommandLineException($"Unknown host {HostConfig.HostString} in config named {HostConfig.Name}"); Host = wasmHost; + + hostArgsList.AddRange(HostConfig.HostArguments); + HostArguments = hostArgsList; } public ProxyOptions ToProxyOptions() diff --git a/src/mono/wasm/host/JSEngineHost.cs b/src/mono/wasm/host/JSEngineHost.cs index fc77d69818a417..cc80f068a18a5c 100644 --- a/src/mono/wasm/host/JSEngineHost.cs +++ b/src/mono/wasm/host/JSEngineHost.cs @@ -37,8 +37,6 @@ public static async Task InvokeAsync(CommonConfiguration commonArgs, private async Task RunAsync() { - string[] engineArgs = Array.Empty(); - string engineBinary = _args.Host switch { WasmHost.V8 => "v8", @@ -79,9 +77,10 @@ private async Task RunAsync() args.Add("--expose_wasm"); } + args.AddRange(_args.CommonConfig.HostArguments); + args.Add(_args.JSPath!); - args.AddRange(engineArgs); if (_args.Host is WasmHost.V8 or WasmHost.JavaScriptCore) { // v8/jsc want arguments to the script separated by "--", others don't diff --git a/src/mono/wasm/host/RuntimeConfigJson.cs b/src/mono/wasm/host/RuntimeConfigJson.cs index 3b7b0564c4d09c..ed698ed8fb3725 100644 --- a/src/mono/wasm/host/RuntimeConfigJson.cs +++ b/src/mono/wasm/host/RuntimeConfigJson.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Collections.Generic; using System.Text.Json; using System.Text.Json.Serialization; @@ -32,6 +33,7 @@ internal sealed record WasmHostProperties( internal sealed record HostConfig(string? Name, [property: JsonPropertyName("host")] string? HostString) { + [property: JsonPropertyName("host-args")] public string[] HostArguments { get; set; } = Array.Empty(); // using an explicit property because the deserializer doesn't like // extension data in the record constructor [property: JsonExtensionData] public Dictionary? Properties { get; set; } From df9fb1d1b69cdb5c86c3d597ca6cc11eaa25b026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Tue, 24 Jan 2023 09:43:34 +0100 Subject: [PATCH 3/3] Drop reallocation of managed dlls. --- src/tasks/WasmAppBuilder/IcallTableGenerator.cs | 2 +- src/tasks/WasmAppBuilder/ManagedToNativeGenerator.cs | 6 +++--- src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/tasks/WasmAppBuilder/IcallTableGenerator.cs b/src/tasks/WasmAppBuilder/IcallTableGenerator.cs index f60f84de156727..2aa9132a7bc9ba 100644 --- a/src/tasks/WasmAppBuilder/IcallTableGenerator.cs +++ b/src/tasks/WasmAppBuilder/IcallTableGenerator.cs @@ -34,7 +34,7 @@ public IcallTableGenerator(Func fixupSymbolName, TaskLoggingHelp // The runtime icall table should be generated using // mono --print-icall-table // - public IEnumerable Generate(string? runtimeIcallTableFile, string[] assemblies, string? outputPath) + public IEnumerable Generate(string? runtimeIcallTableFile, IEnumerable assemblies, string? outputPath) { _icalls.Clear(); _signatures.Clear(); diff --git a/src/tasks/WasmAppBuilder/ManagedToNativeGenerator.cs b/src/tasks/WasmAppBuilder/ManagedToNativeGenerator.cs index 37d7da99d6bf48..9b04d02c670f99 100644 --- a/src/tasks/WasmAppBuilder/ManagedToNativeGenerator.cs +++ b/src/tasks/WasmAppBuilder/ManagedToNativeGenerator.cs @@ -65,7 +65,7 @@ public override bool Execute() private void ExecuteInternal() { - string[] managedAssemblies = FilterOutUnmanagedBinaries(Assemblies); + List managedAssemblies = FilterOutUnmanagedBinaries(Assemblies); var pinvoke = new PInvokeTableGenerator(FixupSymbolName, Log); var icall = new IcallTableGenerator(FixupSymbolName, Log); @@ -116,7 +116,7 @@ public string FixupSymbolName(string name) return fixedName; } - private string[] FilterOutUnmanagedBinaries(string[] assemblies) + private List FilterOutUnmanagedBinaries(string[] assemblies) { List managedAssemblies = new(assemblies.Length); foreach (string asmPath in Assemblies) @@ -141,7 +141,7 @@ private string[] FilterOutUnmanagedBinaries(string[] assemblies) managedAssemblies.Add(asmPath); } - return managedAssemblies.ToArray(); + return managedAssemblies; } private static bool IsManagedAssembly(string filePath) diff --git a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs index 389c7f7bd5b3a8..b8ecfa7f8438eb 100644 --- a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs +++ b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs @@ -24,7 +24,7 @@ public PInvokeTableGenerator(Func fixupSymbolName, TaskLoggingHe _fixupSymbolName = fixupSymbolName; } - public IEnumerable Generate(string[] pinvokeModules, string[] assemblies, string outputPath) + public IEnumerable Generate(string[] pinvokeModules, IEnumerable assemblies, string outputPath) { var modules = new Dictionary(); foreach (var module in pinvokeModules) @@ -37,6 +37,7 @@ public IEnumerable Generate(string[] pinvokeModules, string[] assemblies var resolver = new PathAssemblyResolver(assemblies); using var mlc = new MetadataLoadContext(resolver, "System.Private.CoreLib"); + foreach (var asmPath in assemblies) { if (!File.Exists(asmPath))