From c3a89199594978c427c63096b8a6b099ea145c5a Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Tue, 10 Aug 2021 03:28:35 -0400 Subject: [PATCH 01/23] Update tasks to support incremental build MonoAOTCompiler: - Compiles assemblies to .bc files. - Hashes for the .bc files are stored in a json cache file. - And uses .depfile generated by mono-aot-cross, to figure out the if any of the dependencies have changed - Writes out the actual .bc file only if the assembly, or it's dependencies changed EmccCompile.cs: Support a `%(Dependencies)` metadata on the source files, to compile the files only when needed. --- src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 324 +++++++++++++++--- .../AotCompilerTask/MonoAOTCompiler.csproj | 1 + src/tasks/Common/Utils.cs | 25 ++ src/tasks/WasmAppBuilder/EmccCompile.cs | 125 ++++++- .../WasmAppBuilder/IcallTableGenerator.cs | 14 +- .../WasmAppBuilder/PInvokeTableGenerator.cs | 17 +- src/tasks/WasmAppBuilder/WasmAppBuilder.cs | 6 +- 7 files changed, 436 insertions(+), 76 deletions(-) diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index ab27947c15a7a2..ddb359609623bc 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -9,10 +9,13 @@ using System.Linq; using System.Reflection.Metadata; using System.Text; +using System.Text.Json; +using System.Threading; using System.Threading.Tasks; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using System.Reflection.PortableExecutable; +using System.Text.Json.Serialization; public class MonoAOTCompiler : Microsoft.Build.Utilities.Task { @@ -182,16 +185,25 @@ public class MonoAOTCompiler : Microsoft.Build.Utilities.Task /// public string? LLVMDebug { get; set; } = "nodebug"; + [NotNull] + [Required] + public string? CacheFilePath { get; set; } + [Output] public string[]? FileWrites { get; private set; } private List _fileWrites = new(); - private ConcurrentBag compiledAssemblies = new ConcurrentBag(); + private ConcurrentDictionary compiledAssemblies = new(); + private MonoAotMode parsedAotMode; private MonoAotOutputType parsedOutputType; private MonoAotLibraryFormat parsedLibraryFormat; private MonoAotModulesTableLanguage parsedAotModulesTableLanguage; + private CompilerCache? _newCache; + private CompilerCache? _oldCache; + private int _numCompiled; + private int _totalNumAssemblies; public override bool Execute() { @@ -285,7 +297,7 @@ public override bool Execute() // AOT modules for static linking, needs the aot modules table UseStaticLinking = true; - if (!GenerateAotModulesTable(Assemblies, Profilers)) + if (!GenerateAotModulesTable(Assemblies, Profilers, AotModulesTablePath)) return false; } @@ -308,22 +320,65 @@ public override bool Execute() if (AdditionalAssemblySearchPaths != null) monoPaths = string.Join(Path.PathSeparator.ToString(), AdditionalAssemblySearchPaths); - if (DisableParallelAot) + if (File.Exists(CacheFilePath)) + { + _oldCache = (CompilerCache?)JsonSerializer.Deserialize(File.ReadAllText(CacheFilePath!), + typeof(CompilerCache), + new JsonSerializerOptions()); + _oldCache ??= new(); + } + else + { + _oldCache = new(); + } + + _newCache = new(); + foreach (var assemblyItem in Assemblies) + { + string newHash = Utils.ComputeHash(assemblyItem.GetMetadata("FullPath")); + _newCache.AssemblyHashes[assemblyItem.ItemSpec] = newHash; + } + + //FIXME: check the nothing changed at all case + + _totalNumAssemblies = Assemblies.Length; + int allowedParallelism = Math.Min(Assemblies.Length, Environment.ProcessorCount); + if (BuildEngine is IBuildEngine9 be9) + allowedParallelism = be9.RequestCores(allowedParallelism); + + if (DisableParallelAot || allowedParallelism == 1) { foreach (var assemblyItem in Assemblies) { - if (!PrecompileLibrary(assemblyItem, monoPaths)) + if (!PrecompileLibrarySerial(assemblyItem, monoPaths)) return !Log.HasLoggedErrors; } } else { - Parallel.ForEach(Assemblies, - new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }, - assemblyItem => PrecompileLibrary(assemblyItem, monoPaths)); + ParallelLoopResult result = Parallel.ForEach( + Assemblies, + new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism }, + (assemblyItem, state) => PrecompileLibraryParallel(assemblyItem, monoPaths, state)); + + if (!result.IsCompleted) + { + if (!Log.HasLoggedErrors) + Log.LogError("Unknown failed occured while compiling"); + + return false; + } } - CompiledAssemblies = compiledAssemblies.ToArray(); + int numUnchanged = _totalNumAssemblies - _numCompiled; + if (numUnchanged > 0 && numUnchanged != _totalNumAssemblies) + Log.LogMessage(MessageImportance.High, $"[{numUnchanged}/{_totalNumAssemblies}] skipped unchanged assemblies."); + + var json = JsonSerializer.Serialize (_newCache, new JsonSerializerOptions { WriteIndented = true }); + File.WriteAllText(CacheFilePath!, json); + _fileWrites.Add(CacheFilePath!); + + CompiledAssemblies = ConvertAssembliesDictToOrderedList(compiledAssemblies, Assemblies).ToArray(); FileWrites = _fileWrites.ToArray(); return !Log.HasLoggedErrors; @@ -351,8 +406,6 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) processArgs.AddRange(p.Split(new char[]{ ';' }, StringSplitOptions.RemoveEmptyEntries)); } - Log.LogMessage(MessageImportance.Low, $"[AOT] {assembly}"); - processArgs.Add("--debug"); // add LLVM options @@ -401,12 +454,15 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) aotArgs.Add("dedup-skip"); } + string tmpOutputFileName; + // compute output mode and file names if (parsedAotMode == MonoAotMode.LLVMOnly || parsedAotMode == MonoAotMode.LLVMOnlyInterp) { aotArgs.Add("llvmonly"); string llvmBitcodeFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.bc")); + tmpOutputFileName = llvmBitcodeFile + ".tmp"; aotAssembly.SetMetadata("LlvmBitcodeFile", llvmBitcodeFile); if (parsedAotMode == MonoAotMode.LLVMOnlyInterp) @@ -417,11 +473,11 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) if (parsedOutputType == MonoAotOutputType.AsmOnly) { aotArgs.Add("asmonly"); - aotArgs.Add($"llvm-outfile={llvmBitcodeFile}"); + aotArgs.Add($"llvm-outfile={tmpOutputFileName}"); } else { - aotArgs.Add($"outfile={llvmBitcodeFile}"); + aotArgs.Add($"outfile={tmpOutputFileName}"); } } else @@ -441,39 +497,54 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) aotArgs.Add("interp"); } - if (parsedOutputType == MonoAotOutputType.ObjectFile) + switch (parsedOutputType) { - string objectFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.o")); - aotArgs.Add($"outfile={objectFile}"); - aotAssembly.SetMetadata("ObjectFile", objectFile); - } - else if (parsedOutputType == MonoAotOutputType.AsmOnly) - { - aotArgs.Add("asmonly"); + case MonoAotOutputType.ObjectFile: + { + string objectFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.o")); + tmpOutputFileName = objectFile + ".tmp"; + aotArgs.Add($"outfile={tmpOutputFileName}"); + aotAssembly.SetMetadata("ObjectFile", objectFile); + } + break; - string assemblerFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.s")); - aotArgs.Add($"outfile={assemblerFile}"); - aotAssembly.SetMetadata("AssemblerFile", assemblerFile); - } - else if (parsedOutputType == MonoAotOutputType.Library) - { - string extension = parsedLibraryFormat switch { - MonoAotLibraryFormat.Dll => ".dll", - MonoAotLibraryFormat.Dylib => ".dylib", - MonoAotLibraryFormat.So => ".so", - _ => throw new ArgumentOutOfRangeException() - }; - string libraryFileName = $"{LibraryFilePrefix}{assemblyFilename}{extension}"; - string libraryFilePath = Path.Combine(OutputDir, libraryFileName); - - aotArgs.Add($"outfile={libraryFilePath}"); - aotAssembly.SetMetadata("LibraryFile", libraryFilePath); + case MonoAotOutputType.AsmOnly: + { + aotArgs.Add("asmonly"); + + string assemblerFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.s")); + tmpOutputFileName = assemblerFile + ".tmp"; + aotArgs.Add($"outfile={tmpOutputFileName}"); + aotAssembly.SetMetadata("AssemblerFile", assemblerFile); + } + break; + + case MonoAotOutputType.Library: + { + string extension = parsedLibraryFormat switch { + MonoAotLibraryFormat.Dll => ".dll", + MonoAotLibraryFormat.Dylib => ".dylib", + MonoAotLibraryFormat.So => ".so", + _ => throw new ArgumentOutOfRangeException() + }; + string libraryFileName = $"{LibraryFilePrefix}{assemblyFilename}{extension}"; + string libraryFilePath = Path.Combine(OutputDir, libraryFileName); + tmpOutputFileName = libraryFilePath + ".tmp"; + + aotArgs.Add($"outfile={tmpOutputFileName}"); + aotAssembly.SetMetadata("LibraryFile", libraryFilePath); + } + break; + + default: + throw new Exception($"Bug: Unhandled MonoAotOutputType: {parsedAotMode}"); } if (UseLLVM) { string llvmObjectFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll-llvm.o")); - aotArgs.Add($"llvm-outfile={llvmObjectFile}"); + tmpOutputFileName = llvmObjectFile; + aotArgs.Add($"llvm-outfile={tmpOutputFileName}"); aotAssembly.SetMetadata("LlvmObjectFile", llvmObjectFile); } } @@ -496,6 +567,9 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) aotArgs.Add($"profile={AotProfilePath},profile-only"); } + string depfile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".depfile")); + aotArgs.Add($"depfile={depfile}"); + // we need to quote the entire --aot arguments here to make sure it is parsed // on Windows as one argument. Otherwise it will be split up into multiple // values, which wont work. @@ -545,14 +619,6 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) string workingDir = assemblyDir; - // Log the command in a compact format which can be copy pasted - { - StringBuilder envStr = new StringBuilder(string.Empty); - foreach (KeyValuePair kvp in envVariables) - envStr.Append($"{kvp.Key}={kvp.Value} "); - Log.LogMessage(MessageImportance.Low, $"{msgPrefix}Exec (with response file contents expanded) in {workingDir}: {envStr}{CompilerBinaryPath} {responseFileContent}"); - } - try { // run the AOT compiler @@ -561,12 +627,24 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) $"--response=\"{responseFilePath}\"", envVariables, workingDir, - silent: false, + silent: true, debugMessageImportance: MessageImportance.Low, label: Path.GetFileName(assembly)); + + var importance = exitCode == 0 ? MessageImportance.Low : MessageImportance.High; + // Log the command in a compact format which can be copy pasted + { + StringBuilder envStr = new StringBuilder(string.Empty); + foreach (KeyValuePair kvp in envVariables) + envStr.Append($"{kvp.Key}={kvp.Value} "); + Log.LogMessage(importance, $"{msgPrefix}Exec (with response file contents expanded) in {workingDir}: {envStr}{CompilerBinaryPath} {responseFileContent}"); + } + + Log.LogMessage(importance, output); + if (exitCode != 0) { - Log.LogError($"Precompiling failed for {assembly}: {output}"); + Log.LogError($"Precompiling failed for {assembly}"); return false; } } @@ -577,13 +655,127 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) return false; } + if (!File.Exists(tmpOutputFileName)) + { + Log.LogError($"Precompiling failed for {assembly}. Could not find output file {tmpOutputFileName}"); + return false; + } + + if (HaveAsmDependenciesChanged(assembly, depfile, out string? cause)) + { + if (Path.GetExtension(tmpOutputFileName) == ".tmp") + { + string finalFileName = Path.GetFileNameWithoutExtension(tmpOutputFileName); + + if (File.Exists(finalFileName)) + File.Delete(finalFileName); + + File.Copy(tmpOutputFileName, finalFileName); + + Log.LogMessage(MessageImportance.Low, $"[AOT] {assembly} because {cause}"); + _fileWrites.Add(finalFileName); + + int count = Interlocked.Increment(ref _numCompiled); + Log.LogMessage(MessageImportance.High, $"[{count}/{_totalNumAssemblies}] {assemblyFilename} -> {Path.GetFileName(finalFileName)}"); + } + else + { + Log.LogWarning($"Bug: expected tmpOutputFileName to end in .tmp: {tmpOutputFileName}"); + } + } + else + { + Log.LogMessage(MessageImportance.Low, $"Skipping precompiling assembly {assembly} because it has unchanged hash."); + } + File.Delete(responseFilePath); + File.Delete(tmpOutputFileName); + File.Delete(depfile); - compiledAssemblies.Add(aotAssembly); + compiledAssemblies.GetOrAdd(aotAssembly.ItemSpec, aotAssembly); return true; } - private bool GenerateAotModulesTable(ITaskItem[] assemblies, string[]? profilers) + private bool PrecompileLibrarySerial(ITaskItem assemblyItem, string? monoPaths) + { + try + { + if (!PrecompileLibrary(assemblyItem, monoPaths)) + return !Log.HasLoggedErrors; + return true; + } + catch (Exception ex) + { + if (Log.HasLoggedErrors) + Log.LogMessage(MessageImportance.Low, $"Precompile failed for {assemblyItem}: {ex}"); + else + Log.LogError($"Precompile failed for {assemblyItem}: {ex}"); + + return false; + } + } + + private void PrecompileLibraryParallel(ITaskItem assemblyItem, string? monoPaths, ParallelLoopState state) + { + try + { + if (!PrecompileLibrary(assemblyItem, monoPaths)) + state.Break(); + } + catch (Exception ex) + { + if (Log.HasLoggedErrors) + Log.LogMessage(MessageImportance.Low, $"Precompile failed for {assemblyItem}: {ex}"); + else + Log.LogError($"Precompile failed for {assemblyItem}: {ex}"); + state.Break(); + } + } + + private bool HaveAsmDependenciesChanged(string assemblyFileName, string depfile, [NotNullWhen(true)] out string? cause) + { + cause = null; + if (!File.Exists(depfile)) + { + cause = $"couldn't find depfile {depfile}"; + return true; + } + + string deps = File.ReadAllText(depfile); + string[] parts = deps.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); + if (parts.Length < 2) + { + cause = $"{depfile} had invalid format"; + return true; + } + + Log.LogMessage(MessageImportance.Low, $"{assemblyFileName} depends on {deps}"); + string? changedAsm = parts.Skip(1).FirstOrDefault(asm => HasHashChanged(asm.Trim())); + if (changedAsm != null) + cause = changedAsm == assemblyFileName ? "it changed" : $"dependency {changedAsm} changed."; + + return changedAsm != null; + } + + private bool HasHashChanged(string assemblyFileName) + { + if (!_oldCache!.AssemblyHashes.TryGetValue(assemblyFileName, out string? oldHash)) + { + // we haven't seen this file before + return true; + } + + if (!_newCache!.AssemblyHashes.TryGetValue(assemblyFileName, out string? newHash)) + { + // um should not happen + throw new Exception($"BUG? can't find new hash for {assemblyFileName}"); + } + + var res = oldHash != newHash; + return res; + } + + private bool GenerateAotModulesTable(ITaskItem[] assemblies, string[]? profilers, string outputFile) { var symbols = new List(); foreach (var asm in assemblies) @@ -602,11 +794,11 @@ private bool GenerateAotModulesTable(ITaskItem[] assemblies, string[]? profilers symbols.Add($"mono_aot_module_{symbolName}_info"); } - Directory.CreateDirectory(Path.GetDirectoryName(AotModulesTablePath!)!); + Directory.CreateDirectory(Path.GetDirectoryName(outputFile)!); - using (var writer = File.CreateText(AotModulesTablePath!)) + string tmpAotModulesTablePath = Path.GetTempFileName(); + using (var writer = File.CreateText(tmpAotModulesTablePath)) { - _fileWrites.Add(AotModulesTablePath!); if (parsedAotModulesTableLanguage == MonoAotModulesTableLanguage.C) { writer.WriteLine("#include "); @@ -664,7 +856,12 @@ private bool GenerateAotModulesTable(ITaskItem[] assemblies, string[]? profilers { throw new NotSupportedException(); } - Log.LogMessage(MessageImportance.Low, $"Generated {AotModulesTablePath}"); + } + + if (Utils.CopyIfDifferent(tmpAotModulesTablePath, outputFile)) + { + _fileWrites.Add(outputFile); + Log.LogMessage(MessageImportance.Low, $"Generated {outputFile}"); } return true; @@ -695,6 +892,19 @@ private bool TryGetAssemblyName(string asmPath, [NotNullWhen(true)] out string? return false; } } + + private IList ConvertAssembliesDictToOrderedList(ConcurrentDictionary dict, ITaskItem[] items) + { + List outItems = new(items.Length); + foreach (ITaskItem item in items) + { + if (!dict.TryGetValue(item.ItemSpec, out ITaskItem? dictItem)) + throw new LogAsErrorException($"Bug: Could not find item in the dict with key {item.ItemSpec}"); + + outItems.Add(dictItem); + } + return outItems; + } } public enum MonoAotMode @@ -727,3 +937,9 @@ public enum MonoAotModulesTableLanguage C, ObjC } + +internal class CompilerCache +{ + [JsonPropertyName("assembly_hashes")] + public ConcurrentDictionary AssemblyHashes { get; set; } = new(); +} diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.csproj b/src/tasks/AotCompilerTask/MonoAOTCompiler.csproj index 338ba17ab82072..6371df56223568 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.csproj +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.csproj @@ -20,6 +20,7 @@ + diff --git a/src/tasks/Common/Utils.cs b/src/tasks/Common/Utils.cs index 1d04c4d27dcb0c..5887ea18fa37b9 100644 --- a/src/tasks/Common/Utils.cs +++ b/src/tasks/Common/Utils.cs @@ -6,6 +6,7 @@ using System.Diagnostics; using System.IO; using System.Runtime.InteropServices; +using System.Security.Cryptography; using System.Text; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; @@ -205,6 +206,30 @@ internal static string CreateTemporaryBatchFile(string command) return file; } + public static bool CopyIfDifferent(string src, string dst, bool useHash=false) + { + if (!File.Exists(src)) + throw new ArgumentException($"Cannot find {src} file to copy", nameof(src)); + + bool areDifferent = !File.Exists(dst) || + (useHash && Utils.ComputeHash(src) != Utils.ComputeHash(dst)) || + (File.ReadAllText(src) != File.ReadAllText(dst)); + + if (areDifferent) + File.Copy(src, dst, true); + + return areDifferent; + } + + public static string ComputeHash(string filepath) + { + using var stream = File.OpenRead(filepath); + using HashAlgorithm hashAlgorithm = SHA512.Create(); + + byte[] hash = hashAlgorithm.ComputeHash(stream); + return Convert.ToBase64String(hash); + } + #if NETCOREAPP public static void DirectoryCopy(string sourceDir, string destDir, Func? predicate=null) { diff --git a/src/tasks/WasmAppBuilder/EmccCompile.cs b/src/tasks/WasmAppBuilder/EmccCompile.cs index eed8bb52690696..0c85b89c6ad2c0 100644 --- a/src/tasks/WasmAppBuilder/EmccCompile.cs +++ b/src/tasks/WasmAppBuilder/EmccCompile.cs @@ -8,6 +8,7 @@ using System.IO; using System.Linq; using System.Text; +using System.Threading; using System.Threading.Tasks; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; @@ -39,6 +40,8 @@ public class EmccCompile : Microsoft.Build.Utilities.Task public ITaskItem[]? OutputFiles { get; private set; } private string? _tempPath; + private int _totalFiles; + private int _numCompiled; public override bool Execute() { @@ -61,10 +64,42 @@ public override bool Execute() return false; } + _totalFiles = SourceFiles.Length; IDictionary envVarsDict = GetEnvironmentVariablesDict(); ConcurrentBag outputItems = new(); try { + List<(string, string)> filesToCompile = new(); + foreach (ITaskItem srcItem in SourceFiles) + { + string srcFile = srcItem.ItemSpec; + string objFile = srcItem.GetMetadata("ObjectFile"); + string depMetadata = srcItem.GetMetadata("Dependencies"); + string[] depFiles = string.IsNullOrEmpty(depMetadata) + ? Array.Empty() + : depMetadata.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries); + + if (!ShouldCompile(srcFile, objFile, depFiles, out string reason)) + { + Log.LogMessage(MessageImportance.Low, $"Skipping {srcFile} because {reason}."); + } + else + { + Log.LogMessage(MessageImportance.Low, $"Compiling {srcFile} because {reason}."); + filesToCompile.Add((srcFile, objFile)); + } + } + + _numCompiled = SourceFiles.Length - filesToCompile.Count; + if (_numCompiled == _totalFiles) + { + // nothing to do! + return true; + } + + if (_numCompiled > 0) + Log.LogMessage(MessageImportance.High, $"[{_numCompiled}/{SourceFiles.Length}] skipped unchanged files"); + Log.LogMessage(MessageImportance.Low, "Using environment variables:"); foreach (var kvp in envVarsDict) Log.LogMessage(MessageImportance.Low, $"\t{kvp.Key} = {kvp.Value}"); @@ -76,32 +111,37 @@ public override bool Execute() Directory.CreateDirectory(_tempPath); int allowedParallelism = Math.Min(SourceFiles.Length, Environment.ProcessorCount); -#if false // Enable this when we bump msbuild to 16.1.0 if (BuildEngine is IBuildEngine9 be9) allowedParallelism = be9.RequestCores(allowedParallelism); -#endif if (DisableParallelCompile || allowedParallelism == 1) { - foreach (ITaskItem srcItem in SourceFiles) + foreach ((string srcFile, string outFile) in filesToCompile) { - if (!ProcessSourceFile(srcItem)) + if (!ProcessSourceFile(srcFile, outFile)) return false; } } else { - ParallelLoopResult result = Parallel.ForEach(SourceFiles, + ParallelLoopResult result = Parallel.ForEach(filesToCompile, new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism }, - (srcItem, state) => + (toCompile, state) => { - if (!ProcessSourceFile(srcItem)) + if (!ProcessSourceFile(toCompile.Item1, toCompile.Item2)) state.Stop(); }); if (!result.IsCompleted && !Log.HasLoggedErrors) Log.LogError("Unknown failed occured while compiling"); } + + if (!Log.HasLoggedErrors) + { + int numUnchanged = _totalFiles - _numCompiled; + if (numUnchanged > 0) + Log.LogMessage(MessageImportance.High, $"[{numUnchanged}/{_totalFiles}] unchanged."); + } } finally { @@ -112,14 +152,13 @@ public override bool Execute() OutputFiles = outputItems.ToArray(); return !Log.HasLoggedErrors; - bool ProcessSourceFile(ITaskItem srcItem) + bool ProcessSourceFile(string srcFile, string objFile) { - string srcFile = srcItem.ItemSpec; - string objFile = srcItem.GetMetadata("ObjectFile"); - + string tmpObjFile = Path.GetTempFileName(); try { - string command = $"emcc {Arguments} -c -o \"{objFile}\" \"{srcFile}\""; + string command = $"emcc {Arguments} -c -o \"{tmpObjFile}\" \"{srcFile}\""; + var startTime = DateTime.Now; // Log the command in a compact format which can be copy pasted StringBuilder envStr = new StringBuilder(string.Empty); @@ -135,16 +174,24 @@ bool ProcessSourceFile(ITaskItem srcItem) debugMessageImportance: messageImportance, label: Path.GetFileName(srcFile)); + var endTime = DateTime.Now; + var elapsedSecs = (endTime - startTime).TotalSeconds; if (exitCode != 0) { - Log.LogError($"Failed to compile {srcFile} -> {objFile}"); + Log.LogError($"Failed to compile {srcFile} -> {objFile}{Environment.NewLine}{output} [took {elapsedSecs:F}s]"); return false; } + if (!Utils.CopyIfDifferent(tmpObjFile, objFile, useHash: true)) + Log.LogMessage(MessageImportance.Low, $"Did not overwrite {objFile} as the contents are unchanged"); + ITaskItem newItem = new TaskItem(objFile); newItem.SetMetadata("SourceFile", srcFile); outputItems.Add(newItem); + int count = Interlocked.Increment(ref _numCompiled); + Log.LogMessage(MessageImportance.High, $"[{count}/{_totalFiles}] {Path.GetFileName(srcFile)} -> {Path.GetFileName(objFile)} [took {elapsedSecs:F}s]"); + return !Log.HasLoggedErrors; } catch (Exception ex) @@ -152,6 +199,58 @@ bool ProcessSourceFile(ITaskItem srcItem) Log.LogError($"Failed to compile {srcFile} -> {objFile}{Environment.NewLine}{ex.Message}"); return false; } + finally + { + File.Delete(tmpObjFile); + } + } + } + + private bool ShouldCompile(string srcFile, string objFile, string[] depFiles, out string reason) + { + if (!File.Exists(srcFile)) + throw new ArgumentException($"Could not find source file {srcFile}"); + + if (!File.Exists(objFile)) + { + reason = $"output file {objFile} doesn't exist"; + return true; + } + + if (IsNewerThanOutput(srcFile, objFile, out reason)) + return true; + + foreach (string depFile in depFiles) + { + if (IsNewerThanOutput(depFile, objFile, out reason)) + return true; + } + + reason = "everything is up-to-date."; + return false; + + bool IsNewerThanOutput(string inFile, string outFile, out string reason) + { + if (!File.Exists(inFile)) + { + reason = $"Could not find dependency file {inFile} needed for compiling {srcFile} to {outFile}"; + Log.LogWarning(reason); + return true; + } + + DateTime lastWriteTimeSrc= File.GetLastWriteTimeUtc(inFile); + DateTime lastWriteTimeDst= File.GetLastWriteTimeUtc(outFile); + + if (lastWriteTimeSrc > lastWriteTimeDst) + { + reason = $"{inFile} is newer than {outFile}"; + return true; + } + else + { + reason = $"{inFile} is older than {outFile}"; + return false; + } } } diff --git a/src/tasks/WasmAppBuilder/IcallTableGenerator.cs b/src/tasks/WasmAppBuilder/IcallTableGenerator.cs index e0737521dbfa30..592ac56a3e1233 100644 --- a/src/tasks/WasmAppBuilder/IcallTableGenerator.cs +++ b/src/tasks/WasmAppBuilder/IcallTableGenerator.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using System.Text; @@ -19,7 +20,7 @@ public class IcallTableGenerator : Task public string? RuntimeIcallTableFile { get; set; } [Required] public ITaskItem[]? Assemblies { get; set; } - [Required] + [Required, NotNull] public string? OutputPath { get; set; } private List _icalls = new List (); @@ -27,7 +28,6 @@ public class IcallTableGenerator : Task public override bool Execute() { - Log.LogMessage(MessageImportance.Normal, $"Generating icall table to '{OutputPath}'."); GenIcallTable(RuntimeIcallTableFile!, Assemblies!.Select(item => item.ItemSpec).ToArray()); return true; } @@ -50,8 +50,16 @@ public void GenIcallTable(string runtimeIcallTableFile, string[] assemblies) ProcessType(type); } - using (var w = File.CreateText(OutputPath!)) + string tmpFileName = Path.GetTempFileName(); + using (var w = File.CreateText(tmpFileName)) EmitTable (w); + + if (Utils.CopyIfDifferent(tmpFileName, OutputPath)) + Log.LogMessage(MessageImportance.Low, $"Generating icall table to '{OutputPath}'."); + else + Log.LogMessage(MessageImportance.Low, $"Icall table in {OutputPath} is unchanged."); + + File.Delete(tmpFileName); } private void EmitTable (StreamWriter w) diff --git a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs index daf0b8afc56eed..d5a365f55c0285 100644 --- a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs +++ b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using System.Text; @@ -19,14 +20,14 @@ public class PInvokeTableGenerator : Task public ITaskItem[]? Modules { get; set; } [Required] public ITaskItem[]? Assemblies { get; set; } - [Required] + + [Required, NotNull] public string? OutputPath { get; set; } private static char[] s_charsToReplace = new[] { '.', '-', }; public override bool Execute() { - Log.LogMessage(MessageImportance.Normal, $"Generating pinvoke table to '{OutputPath}'."); GenPInvokeTable(Modules!.Select(item => item.ItemSpec).ToArray(), Assemblies!.Select(item => item.ItemSpec).ToArray()); return true; } @@ -49,11 +50,19 @@ public void GenPInvokeTable(string[] pinvokeModules, string[] assemblies) CollectPInvokes(pinvokes, callbacks, type); } - using (var w = File.CreateText(OutputPath!)) + string tmpFileName = Path.GetTempFileName(); + using (var w = File.CreateText(tmpFileName)) { EmitPInvokeTable(w, modules, pinvokes); EmitNativeToInterp(w, callbacks); } + + if (Utils.CopyIfDifferent(tmpFileName, OutputPath)) + Log.LogMessage(MessageImportance.Low, $"Generating pinvoke table to '{OutputPath}'."); + else + Log.LogMessage(MessageImportance.Low, $"PInvoke table in {OutputPath} is unchanged."); + + File.Delete(tmpFileName); } private void CollectPInvokes(List pinvokes, List callbacks, Type type) @@ -119,7 +128,7 @@ private void EmitPInvokeTable(StreamWriter w, Dictionary modules Where(l => l.Module == module && !l.Skip). OrderBy(l => l.EntryPoint). GroupBy(d => d.EntryPoint). - Select (l => "{\"" + l.Key + "\", " + l.Key + "}, // " + string.Join (", ", l.Select(c => c.Method.DeclaringType!.Module!.Assembly!.GetName ()!.Name!).Distinct())); + Select (l => "{\"" + l.Key + "\", " + l.Key + "}, // " + string.Join (", ", l.Select(c => c.Method.DeclaringType!.Module!.Assembly!.GetName ()!.Name!).Distinct().OrderBy(n => n))); foreach (var pinvoke in assemblies_pinvokes) { w.WriteLine (pinvoke); diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs index 46c0174148be25..fd8f489e1c861c 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs +++ b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs @@ -248,12 +248,14 @@ public override bool Execute () config.Extra[name] = valueObject; } - string monoConfigPath = Path.Combine(AppDir, "mono-config.json"); - using (var sw = File.CreateText(monoConfigPath)) + string tmpMonoConfigPath = Path.GetTempFileName(); + using (var sw = File.CreateText(tmpMonoConfigPath)) { var json = JsonSerializer.Serialize (config, new JsonSerializerOptions { WriteIndented = true }); sw.Write(json); } + string monoConfigPath = Path.Combine(AppDir, "mono-config.json"); + Utils.CopyIfDifferent(tmpMonoConfigPath, monoConfigPath); _fileWrites.Add(monoConfigPath); if (ExtraFilesToDeploy != null) From 344deec13d86dcdfcc6d0fad6ee9d6c8f0cfc0ee Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Tue, 10 Aug 2021 04:06:53 -0400 Subject: [PATCH 02/23] Update wasm targets --- src/mono/wasm/build/WasmApp.Native.targets | 115 ++++++++++++++------- src/mono/wasm/build/WasmApp.targets | 15 +-- 2 files changed, 84 insertions(+), 46 deletions(-) diff --git a/src/mono/wasm/build/WasmApp.Native.targets b/src/mono/wasm/build/WasmApp.Native.targets index 3e61a7074d3fb4..5b43374ab16b7f 100644 --- a/src/mono/wasm/build/WasmApp.Native.targets +++ b/src/mono/wasm/build/WasmApp.Native.targets @@ -33,6 +33,10 @@ true + + <_MonoComponent Include="hot_reload;debugger" /> + + @@ -145,6 +149,10 @@ <_WasmICallTablePath>$(_WasmIntermediateOutputPath)icall-table.h <_WasmRuntimeICallTablePath>$(_WasmIntermediateOutputPath)runtime-icall-table.h <_WasmPInvokeTablePath>$(_WasmIntermediateOutputPath)pinvoke-table.h + <_WasmPInvokeHPath>$(_WasmRuntimePackIncludeDir)wasm\pinvoke.h + <_DriverGenCPath>$(_WasmIntermediateOutputPath)driver-gen.c + + <_DriverGenCNeeded Condition="'$(_DriverGenCNeeded)' == '' and '$(RunAOTCompilation)' == 'true'">true <_EmccAssertionLevelDefault>0 <_EmccOptimizationFlagDefault Condition="'$(_WasmDevel)' == 'true'">-O0 -s ASSERTIONS=$(_EmccAssertionLevelDefault) @@ -159,10 +167,15 @@ <_EmccCompileOutputMessageImportance Condition="'$(EmccVerbose)' == 'true'">Normal <_EmccCompileOutputMessageImportance Condition="'$(EmccVerbose)' != 'true'">Low + <_EmccCompileBitcodeRsp>$(_WasmIntermediateOutputPath)emcc-compile-bc.rsp + <_EmccLinkRsp>$(_WasmIntermediateOutputPath)emcc-link.rsp + 536870912 + <_WasmLinkDependencies Remove="@(_WasmLinkDependencies)" /> + <_EmccCommonFlags Include="$(_DefaultEmccFlags)" /> <_EmccCommonFlags Include="$(EmccFlags)" /> <_EmccCommonFlags Include="-s DISABLE_EXCEPTION_CATCHING=0" /> @@ -188,9 +201,20 @@ <_EmccCFlags Include=""-I%(_EmccIncludePaths.Identity)"" /> <_EmccCFlags Include="-g" Condition="'$(WasmNativeDebugSymbols)' == 'true'" /> - <_EmccCFlags Include="$(EmccExtraCFlags)" /> + + <_EmccLDFlags Include="$(EmccLinkOptimizationFlag)" /> + <_EmccLDFlags Include="@(_EmccCommonFlags)" /> + <_EmccLDFlags Include="-s TOTAL_MEMORY=$(EmccTotalMemory)" /> + + <_DriverCDependencies Include="$(_WasmPInvokeHPath);$(_WasmICallTablePath)" /> + <_DriverCDependencies Include="$(_DriverGenCPath)" Condition="'$(_DriverGenCNeeded)' == 'true'" /> + + <_WasmRuntimePackSrcFile Include="$(_WasmRuntimePackSrcDir)pinvoke.c" + Dependencies="$(_WasmPInvokeHPath);$(_WasmPInvokeTablePath)" /> + <_WasmRuntimePackSrcFile Include="$(_WasmRuntimePackSrcDir)driver.c" + Dependencies="@(_DriverCDependencies)" /> + <_WasmRuntimePackSrcFile Include="$(_WasmRuntimePackSrcDir)corebindings.c" /> - <_WasmRuntimePackSrcFile Include="$(_WasmRuntimePackSrcDir)*.c" /> <_WasmRuntimePackSrcFile ObjectFile="$(_WasmIntermediateOutputPath)%(FileName).o" /> <_DotnetJSSrcFile Include="$(_WasmRuntimePackSrcDir)\*.js" /> @@ -226,62 +250,64 @@ OutputPath="$(_WasmICallTablePath)" /> - - - <_WasmSourceFileToCompile Remove="@(_WasmSourceFileToCompile)" /> - <_WasmSourceFileToCompile Include="@(_WasmRuntimePackSrcFile)" /> - + + <_EmBuilder Condition="$([MSBuild]::IsOSPlatform('WINDOWS'))">embuilder.bat <_EmBuilder Condition="!$([MSBuild]::IsOSPlatform('WINDOWS'))">embuilder.py + + <_EmccCFlags Include="$(EmccExtraCFlags)" /> + + + + <_WasmSourceFileToCompile Remove="@(_WasmSourceFileToCompile)" /> + <_WasmSourceFileToCompile Include="@(_WasmRuntimePackSrcFile)" Dependencies="%(_WasmRuntimePackSrcFile.Dependencies);$(_EmccDefaultFlagsRsp);$(_EmccCompileRsp)" /> + - - - - - - <_MonoComponent Include="hot_reload;debugger" /> - - - - + - - - <_EmccLDFlags Include="$(EmccLinkOptimizationFlag)" /> - <_EmccLDFlags Include="@(_EmccCommonFlags)" /> - - <_EmccLDFlags Include="-s TOTAL_MEMORY=$(EmccTotalMemory)" /> - <_EmccLDFlags Include="$(EmccExtraLDFlags)" /> + <_BitCodeFile Dependencies="%(_BitCodeFile.Dependencies);$(_EmccDefaultFlagsRsp);$(_EmccCompileBitcodeRsp)" /> + + + - + <_BitcodeLDFlags Include="@(_EmccLDFlags)" /> + <_BitcodeLDFlags Include="$(EmccExtraBitcodeLDFlags)" /> + + + + + + + <_WasmNativeFileForLinking Include="%(_BitcodeFile.ObjectFile)" /> <_WasmNativeFileForLinking Include="%(_WasmSourceFileToCompile.ObjectFile)" /> @@ -293,25 +319,35 @@ <_EmccLinkStepArgs Include="@(_EmccLDFlags)" /> <_EmccLinkStepArgs Include="--js-library "%(_DotnetJSSrcFile.Identity)"" /> - <_EmccLinkStepArgs Include="--js-library "%(_WasmExtraJSFile.Identity)"" Condition="'%(_WasmExtraJSFile.Kind)' == 'js-library'" /> + <_WasmLinkDependencies Include="@(_DotnetJSSrcFile)" /> + <_EmccLinkStepArgs Include="--js-library "%(_WasmExtraJSFile.Identity)"" Condition="'%(_WasmExtraJSFile.Kind)' == 'js-library'" /> <_EmccLinkStepArgs Include="--pre-js "%(_WasmExtraJSFile.Identity)"" Condition="'%(_WasmExtraJSFile.Kind)' == 'pre-js'" /> <_EmccLinkStepArgs Include="--post-js "%(_WasmExtraJSFile.Identity)"" Condition="'%(_WasmExtraJSFile.Kind)' == 'post-js'" /> + <_WasmLinkDependencies Include="@(_WasmExtraJSFile)" Condition="'%(_WasmExtraJSFile.Kind)' == 'js-library' or '%(_WasmExtraJSFile.Kind)' == 'pre-js' or '%(_WasmExtraJSFile.Kind)' == 'post-js'" /> <_EmccLinkStepArgs Include=""%(_WasmNativeFileForLinking.Identity)"" /> + <_WasmLinkDependencies Include="@(_WasmNativeFileForLinking)" /> + <_EmccLinkStepArgs Include="-o "$(_WasmIntermediateOutputPath)dotnet.js"" /> - + <_WasmLinkDependencies Include="$(_EmccLinkRsp)" /> - - <_EmccLinkRsp>$(_WasmIntermediateOutputPath)emcc-link.rsp - + <_EmccLinkStepArgs Include="$(EmccExtraLDFlags)" /> + + + + - + + @@ -325,16 +361,15 @@ $(EmccExtraCFlags) -DDRIVER_GEN=1 + <_DriverGenCNeeded>true void mono_profiler_init_aot (const char *desc)%3B EMSCRIPTEN_KEEPALIVE void mono_wasm_load_profiler_aot (const char *desc) { mono_profiler_init_aot (desc)%3B } - - <_DriverGenCPath>$(_WasmIntermediateOutputPath)driver-gen.c - + @@ -414,6 +449,7 @@ EMSCRIPTEN_KEEPALIVE void mono_wasm_load_profiler_aot (const char *desc) { mono_ LLVMOnlyInterp + <_AOTCompilerCacheFile>$(_WasmIntermediateOutputPath)aot_compiler_cache.json diff --git a/src/mono/wasm/build/WasmApp.targets b/src/mono/wasm/build/WasmApp.targets index 32c55ee12bf6b0..e5ceb13def292e 100644 --- a/src/mono/wasm/build/WasmApp.targets +++ b/src/mono/wasm/build/WasmApp.targets @@ -107,6 +107,8 @@ <_WasmRuntimePackSrcDir>$([MSBuild]::NormalizeDirectory($(MicrosoftNetCoreAppRuntimePackRidNativeDir), 'src')) <_WasmIntermediateOutputPath>$([MSBuild]::NormalizeDirectory($(IntermediateOutputPath), 'wasm')) + + <_DriverGenCPath>$(_WasmIntermediateOutputPath)driver-gen.c @@ -125,10 +127,11 @@ <_MainAssemblyPath Condition="'%(WasmAssembliesToBundle.FileName)' == $(AssemblyName) and '%(WasmAssembliesToBundle.Extension)' == '.dll' and $(WasmGenerateAppBundle) == 'true'">%(WasmAssembliesToBundle.Identity) <_WasmRuntimeConfigFilePath Condition="$(_MainAssemblyPath) != ''">$([System.IO.Path]::ChangeExtension($(_MainAssemblyPath), '.runtimeconfig.json')) + <_ParsedRuntimeConfigFilePath Condition="'$(_MainAssemblyPath)' != ''">$([System.IO.Path]::GetDirectoryName($(_MainAssemblyPath)))\runtimeconfig.bin - @@ -143,17 +146,15 @@ - - - <_ParsedRuntimeConfigFilePath>$([System.IO.Path]::GetDirectoryName($(_MainAssemblyPath)))\runtimeconfig.bin - - + <_RuntimeConfigReservedProperties Include="RUNTIME_IDENTIFIER"/> <_RuntimeConfigReservedProperties Include="APP_CONTEXT_BASE_DIRECTORY"/> - Date: Tue, 10 Aug 2021 04:08:38 -0400 Subject: [PATCH 03/23] don't pass unncessary args to RunTests --- eng/testing/tests.wasm.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/testing/tests.wasm.targets b/eng/testing/tests.wasm.targets index 8d6829e47b3387..386ef21d204f59 100644 --- a/eng/testing/tests.wasm.targets +++ b/eng/testing/tests.wasm.targets @@ -21,7 +21,7 @@ <_XHarnessArgs Condition="'$(OS)' != 'Windows_NT'">wasm $XHARNESS_COMMAND --app=. --output-directory=$XHARNESS_OUT <_XHarnessArgs Condition="'$(OS)' == 'Windows_NT'">wasm %XHARNESS_COMMAND% --app=. --output-directory=%XHARNESS_OUT% - <_XHarnessArgs Condition="'$(Scenario)' != 'WasmTestOnBrowser'">$(_XHarnessArgs) --engine=$(JSEngine) $(JSEngineArgs) --js-file=runtime.js + <_XHarnessArgs Condition="'$(Scenario)' != 'WasmTestOnBrowser' and '$(Scenario)' != 'BuildWasmApps'">$(_XHarnessArgs) --engine=$(JSEngine) $(JSEngineArgs) --js-file=runtime.js <_XHarnessArgs Condition="'$(BrowserHost)' == 'windows'">$(_XHarnessArgs) --browser=chrome --browser-path=%HELIX_CORRELATION_PAYLOAD%\chrome-win\chrome.exe <_XHarnessArgs Condition="'$(IsFunctionalTest)' == 'true'" >$(_XHarnessArgs) --expected-exit-code=$(ExpectedExitCode) <_XHarnessArgs Condition="'$(WasmXHarnessArgs)' != ''" >$(_XHarnessArgs) $(WasmXHarnessArgs) From a795ef8a6dbece0085948a34bbc8bab398c95ddb Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Tue, 10 Aug 2021 04:11:07 -0400 Subject: [PATCH 04/23] Add tests for incremental builds --- .../Wasm.Build.Tests/BuildAndRunAttribute.cs | 29 +- .../Wasm.Build.Tests/BuildTestBase.cs | 73 ++-- .../Wasm.Build.Tests/HelperExtensions.cs | 19 + .../Wasm.Build.Tests/NativeRebuildTests.cs | 394 ++++++++++++++++++ .../Wasm.Build.Tests/RebuildTests.cs | 18 +- .../SharedBuildPerTestClassFixture.cs | 8 +- 6 files changed, 490 insertions(+), 51 deletions(-) create mode 100644 src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests.cs diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildAndRunAttribute.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildAndRunAttribute.cs index 46e7215505f6ab..c49b19517b3f2a 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildAndRunAttribute.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildAndRunAttribute.cs @@ -20,21 +20,28 @@ namespace Wasm.Build.Tests [AttributeUsage(AttributeTargets.Method, AllowMultiple = true, Inherited = true)] public class BuildAndRunAttribute : DataAttribute { - private bool _aot; - private RunHost _host; - private object?[] _parameters; + private readonly IEnumerable _data; - public BuildAndRunAttribute(bool aot=false, RunHost host = RunHost.All, params object?[] parameters) + public BuildAndRunAttribute(BuildArgs buildArgs, RunHost host = RunHost.All, params object?[] parameters) { - this._aot = aot; - this._host = host; - this._parameters = parameters; + _data = new IEnumerable[] + { + new object?[] { buildArgs }.AsEnumerable(), + } + .AsEnumerable() + .Multiply(parameters) + .WithRunHosts(host) + .UnwrapItemsAsArrays().ToList().Dump(); } - public override IEnumerable GetData(MethodInfo testMethod) - => BuildTestBase.ConfigWithAOTData(_aot) - .Multiply(_parameters) - .WithRunHosts(_host) + public BuildAndRunAttribute(bool aot=false, RunHost host = RunHost.All, params object?[] parameters) + { + _data = BuildTestBase.ConfigWithAOTData(aot) + .Multiply(parameters) + .WithRunHosts(host) .UnwrapItemsAsArrays().ToList().Dump(); + } + + public override IEnumerable GetData(MethodInfo testMethod) => _data; } } diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildTestBase.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildTestBase.cs index 6d08fe8512461e..2c19fa7167296b 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildTestBase.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildTestBase.cs @@ -86,25 +86,29 @@ public BuildTestBase(ITestOutputHelper output, SharedBuildPerTestClassFixture bu - aot but no wrapper - check that AppBundle wasn't generated */ - public static IEnumerable> ConfigWithAOTData(bool aot) - => new IEnumerable[] + public static IEnumerable> ConfigWithAOTData(bool aot, string? config=null) + { + if (config == null) + { + return new IEnumerable[] + { + #if TEST_DEBUG_CONFIG_ALSO + // list of each member data - for Debug+@aot + new object?[] { new BuildArgs("placeholder", "Debug", aot, "placeholder", string.Empty) }.AsEnumerable(), + #endif + // list of each member data - for Release+@aot + new object?[] { new BuildArgs("placeholder", "Release", aot, "placeholder", string.Empty) }.AsEnumerable() + }.AsEnumerable(); + } + else + { + return new IEnumerable[] { -#if TEST_DEBUG_CONFIG_ALSO - // list of each member data - for Debug+@aot - new object?[] { new BuildArgs("placeholder", "Debug", aot, "placeholder", string.Empty) }.AsEnumerable(), -#endif - - // list of each member data - for Release+@aot - new object?[] { new BuildArgs("placeholder", "Release", aot, "placeholder", string.Empty) }.AsEnumerable() - }.AsEnumerable(); - - public static IEnumerable BuildAndRunData(bool aot = false, - RunHost host = RunHost.All, - params object[] parameters) - => ConfigWithAOTData(aot) - .Multiply(parameters) - .WithRunHosts(host) - .UnwrapItemsAsArrays(); + new object?[] { new BuildArgs("placeholder", config, aot, "placeholder", string.Empty) }.AsEnumerable() + }; + } + } + protected string RunAndTestWasmApp(BuildArgs buildArgs, RunHost host, @@ -271,7 +275,8 @@ protected static BuildArgs ExpandBuildArgs(BuildArgs buildArgs, string extraProp bool hasIcudt = true, bool useCache = true, bool expectSuccess = true, - bool createProject = true) + bool createProject = true, + string? verbosity=null) { if (useCache && _buildContext.TryGetBuildFor(buildArgs, out BuildProduct? product)) { @@ -310,7 +315,7 @@ protected static BuildArgs ExpandBuildArgs(BuildArgs buildArgs, string extraProp _testOutput.WriteLine($"Binlog path: {logFilePath}"); Console.WriteLine($"Binlog path: {logFilePath}"); sb.Append($" /bl:\"{logFilePath}\" /nologo"); - sb.Append($" /v:diag /fl /flp:\"v:diag,LogFile={logFilePath}.log\" /v:minimal"); + sb.Append($" /fl /flp:\"v:diag,LogFile={logFilePath}.log\" /v:{verbosity ?? "minimal"}"); if (buildArgs.ExtraBuildArgs != null) sb.Append($" {buildArgs.ExtraBuildArgs} "); @@ -418,8 +423,8 @@ protected static void AssertFilesExist(string dir, string[] filenames, string? l { Assert.True(File.Exists(path), label != null - ? $"{label}: {path} doesn't exist" - : $"{path} doesn't exist"); + ? $"{label}: File exists: {path}" + : $"File exists: {path}"); } else { @@ -443,9 +448,9 @@ protected static void AssertFile(string file0, string file1, string? label=null, FileInfo finfo1 = new(file1); if (same) - Assert.True(finfo0.Length == finfo1.Length, $"{label}: File sizes don't match for {file0} ({finfo0.Length}), and {file1} ({finfo1.Length})"); + Assert.True(finfo0.Length == finfo1.Length, $"{label}:{Environment.NewLine} File sizes don't match for {file0} ({finfo0.Length}), and {file1} ({finfo1.Length})"); else - Assert.True(finfo0.Length != finfo1.Length, $"{label}: File sizes should not match for {file0} ({finfo0.Length}), and {file1} ({finfo1.Length})"); + Assert.True(finfo0.Length != finfo1.Length, $"{label}:{Environment.NewLine} File sizes should not match for {file0} ({finfo0.Length}), and {file1} ({finfo1.Length})"); } protected (int exitCode, string buildOutput) AssertBuild(string args, string label="build", bool expectSuccess=true, IDictionary? envVars=null, int? timeoutMs=null) @@ -466,6 +471,13 @@ protected string GetBinDir(string config, string targetFramework=s_targetFramewo return Path.Combine(dir!, "bin", config, targetFramework, "browser-wasm"); } + protected string GetObjDir(string config, string targetFramework=s_targetFramework, string? baseDir=null) + { + var dir = baseDir ?? _projectDir; + Assert.NotNull(dir); + return Path.Combine(dir!, "obj", config, targetFramework, "browser-wasm"); + } + public static (int exitCode, string buildOutput) RunProcess(string path, ITestOutputHelper _testOutput, string args = "", @@ -583,11 +595,8 @@ void LogData(string label, string? message) public void Dispose() { - if (s_skipProjectCleanup || !_enablePerTestCleanup) - return; - - if (_projectDir != null) - _buildContext.RemoveFromCache(_projectDir); + if (_projectDir != null && _enablePerTestCleanup) + _buildContext.RemoveFromCache(_projectDir, keepDir: s_skipProjectCleanup); } private static string GetEnvironmentVariableOrDefault(string envVarName, string defaultValue) @@ -605,6 +614,10 @@ public static int Main() }"; } - public record BuildArgs(string ProjectName, string Config, bool AOT, string ProjectFileContents, string? ExtraBuildArgs); + public record BuildArgs(string ProjectName, + string Config, + bool AOT, + string ProjectFileContents, + string? ExtraBuildArgs); public record BuildProduct(string ProjectDir, string LogFile, bool Result); } diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/HelperExtensions.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/HelperExtensions.cs index e3d1ca3dd37b2a..a70a51bb10bbd8 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/HelperExtensions.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/HelperExtensions.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Linq; using System.IO; +using System.Text; #nullable enable @@ -92,5 +93,23 @@ public static class HelperExtensions .Append((object?)runId)); }); } + + public static void UpdateTo(this IDictionary dict, bool unchanged, params string[] filenames) + { + foreach (var filename in filenames) + { + if (!dict.TryGetValue(filename, out var oldValue)) + { + StringBuilder sb = new(); + sb.AppendLine($"Cannot find key named {filename} in the dict. Existing ones:"); + foreach (var kvp in dict) + sb.AppendLine($"[{kvp.Key}] = [{kvp.Value}]"); + + throw new KeyNotFoundException(sb.ToString()); + } + + dict[filename] = (oldValue.fullPath, unchanged); + } + } } } diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests.cs new file mode 100644 index 00000000000000..b20935abe2f48d --- /dev/null +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests.cs @@ -0,0 +1,394 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Collections.Generic; +using System.Linq; +using Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; +using System.Text; + +#nullable enable + +namespace Wasm.Build.Tests +{ + // TODO: test for runtime components + public class NativeRebuildTests : BuildTestBase + { + public NativeRebuildTests(ITestOutputHelper output, SharedBuildPerTestClassFixture buildContext) + : base(output, buildContext) + { + _enablePerTestCleanup = true; + } + + public static IEnumerable NativeBuildData() + { + List data = new(); + // relinking + data.AddRange(GetData(aot: false, nativeRelinking: true, invariant: false)); + data.AddRange(GetData(aot: false, nativeRelinking: true, invariant: true)); + + // aot + data.AddRange(GetData(aot: true, nativeRelinking: false, invariant: false)); + data.AddRange(GetData(aot: true, nativeRelinking: false, invariant: true)); + + return data; + + IEnumerable GetData(bool aot, bool nativeRelinking, bool invariant) + => ConfigWithAOTData(aot) + .Multiply(new object[] { nativeRelinking, invariant }) + .WithRunHosts(RunHost.V8) + .UnwrapItemsAsArrays().ToList().Dump(); + } + + [Theory] + [MemberData(nameof(NativeBuildData))] + public void NoOpRebuildForNativeBuilds(BuildArgs buildArgs, bool nativeRelink, bool invariant, RunHost host, string id) + { + buildArgs = buildArgs with { ProjectName = $"rebuild_noop_{buildArgs.Config}" }; + (buildArgs, BuildPaths paths) = FirstNativeBuild(s_mainReturns42, nativeRelink: nativeRelink, invariant: invariant, buildArgs, id); + + var pathsDict = GetFilesTable(buildArgs, paths, unchanged: true); + var originalStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + Rebuild(nativeRelink, invariant, buildArgs, id); + var newStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + CompareStat(originalStat, newStat, pathsDict.Values); + RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); + } + + [Theory] + [MemberData(nameof(NativeBuildData))] + public void SimpleStringChangeInSource(BuildArgs buildArgs, bool nativeRelink, bool invariant, RunHost host, string id) + { + buildArgs = buildArgs with { ProjectName = $"rebuild_simple_{buildArgs.Config}" }; + (buildArgs, BuildPaths paths) = FirstNativeBuild(s_mainReturns42, nativeRelink, invariant: invariant, buildArgs, id); + + string mainAssembly = $"{buildArgs.ProjectName}.dll"; + var pathsDict = GetFilesTable(buildArgs, paths, unchanged: true); + pathsDict.UpdateTo(unchanged: false, mainAssembly); + pathsDict.UpdateTo(unchanged: !buildArgs.AOT, "dotnet.wasm", "dotnet.js"); + + if (buildArgs.AOT) + pathsDict.UpdateTo(unchanged: false, $"{mainAssembly}.bc", $"{mainAssembly}.o"); + + var originalStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + // Changes + string mainResults55 = @" + public class TestClass { + public static int Main() + { + return 55; + } + }"; + File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), mainResults55); + + // Rebuild + Rebuild(nativeRelink, invariant, buildArgs, id); + var newStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + CompareStat(originalStat, newStat, pathsDict.Values); + RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 55, host: host, id: id); + } + + [Theory] + [MemberData(nameof(NativeBuildData))] + public void ReferenceNewAssembly(BuildArgs buildArgs, bool nativeRelink, bool invariant, RunHost host, string id) + { + buildArgs = buildArgs with { ProjectName = $"rebuild_tasks_{buildArgs.Config}" }; + (buildArgs, BuildPaths paths) = FirstNativeBuild(s_mainReturns42, nativeRelink, invariant: invariant, buildArgs, id); + + var pathsDict = GetFilesTable(buildArgs, paths, unchanged: false); + pathsDict.UpdateTo(unchanged: true, "corebindings.o"); + if (!buildArgs.AOT) // relinking + pathsDict.UpdateTo(unchanged: true, "driver-gen.c"); + + var originalStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + string programText = + @$" + using System; + using System.Text.Json; + public class Test + {{ + public static int Main() + {{" + + @" string json = ""{ \""name\"": \""value\"" }"";" + + @" var jdoc = JsonDocument.Parse($""{json}"", new JsonDocumentOptions());" + + @$" Console.WriteLine($""json: {{jdoc}}""); + return 42; + }} + }}"; + File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText); + + Rebuild(nativeRelink, invariant, buildArgs, id); + var newStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + CompareStat(originalStat, newStat, pathsDict.Values); + RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); + } + + public static IEnumerable FlagsChangesForNativeRelinkingData(bool aot) + => ConfigWithAOTData(aot, config: "Release").Multiply( + new object[] { /*cflags*/ "/p:EmccExtraCFlags=-g", /*ldflags*/ "" }, + new object[] { /*cflags*/ "", /*ldflags*/ "/p:EmccExtraLDFlags=-g" }, + new object[] { /*cflags*/ "/p:EmccExtraCFlags=-g", /*ldflags*/ "/p:EmccExtraLDFlags=-g" } + ).WithRunHosts(RunHost.V8).UnwrapItemsAsArrays().Dump(); + + [Theory] + [MemberData(nameof(FlagsChangesForNativeRelinkingData), parameters: /*aot*/ false)] + [MemberData(nameof(FlagsChangesForNativeRelinkingData), parameters: /*aot*/ true)] + public void ExtraEmccFlagsSetButNoRealChange(BuildArgs buildArgs, string extraCFlags, string extraLDFlags, RunHost host, string id) + { + buildArgs = buildArgs with { ProjectName = $"rebuild_flags_{buildArgs.Config}" }; + (buildArgs, BuildPaths paths) = FirstNativeBuild(s_mainReturns42, nativeRelink: true, invariant: false, buildArgs, id); + var pathsDict = GetFilesTable(buildArgs, paths, unchanged: true); + if (extraLDFlags.Length > 0) + pathsDict.UpdateTo(unchanged: false, "dotnet.wasm", "dotnet.js"); + + var originalStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + // Rebuild + + string mainAssembly = $"{buildArgs.ProjectName}.dll"; + string extraBuildArgs = $" {extraCFlags} {extraLDFlags}"; + string output = Rebuild(nativeRelink: true, invariant: false, buildArgs, id, extraBuildArgs: extraBuildArgs, verbosity: "normal"); + + var newStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + CompareStat(originalStat, newStat, pathsDict.Values); + + // cflags: pinvoke get's compiled, but doesn't overwrite pinvoke.o + // and thus doesn't cause relinking + AssertSubstring("pinvoke.c -> pinvoke.o", output, contains: extraCFlags.Length > 0); + + // ldflags: link step args change, so it should trigger relink + AssertSubstring("wasm-opt", output, contains: extraLDFlags.Length > 0); + + if (buildArgs.AOT) + { + // ExtraEmccLDFlags does not affect .bc files + Assert.DoesNotContain("Compiling assembly bitcode files", output); + } + + string runOutput = RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); + AssertSubstring($"Found statically linked AOT module '{Path.GetFileNameWithoutExtension(mainAssembly)}'", runOutput, + contains: buildArgs.AOT); + } + + public static IEnumerable FlagsOnlyChangeData(bool aot) + => ConfigWithAOTData(aot, config: "Release").Multiply( + new object[] { /*cflags*/ "/p:EmccCompileOptimizationFlag=-O1", /*ldflags*/ "" }, + new object[] { /*cflags*/ "", /*ldflags*/ "/p:EmccLinkOptimizationFlag=-O0" } + ).WithRunHosts(RunHost.V8).UnwrapItemsAsArrays().Dump(); + + [Theory] + [MemberData(nameof(FlagsOnlyChangeData), parameters: /*aot*/ false)] + [MemberData(nameof(FlagsOnlyChangeData), parameters: /*aot*/ true)] + public void OptimizationFlagChange(BuildArgs buildArgs, string cflags, string ldflags, RunHost host, string id) + { + // force _WasmDevel=false, so we don't get -O0 + buildArgs = buildArgs with { ProjectName = $"rebuild_flags_{buildArgs.Config}", ExtraBuildArgs = "/p:_WasmDevel=false" }; + (buildArgs, BuildPaths paths) = FirstNativeBuild(s_mainReturns42, nativeRelink: true, invariant: false, buildArgs, id); + + string mainAssembly = $"{buildArgs.ProjectName}.dll"; + var pathsDict = GetFilesTable(buildArgs, paths, unchanged: false); + pathsDict.UpdateTo(unchanged: true, mainAssembly, "icall-table.h", "pinvoke-table.h", "driver-gen.c"); + if (cflags.Length == 0) + pathsDict.UpdateTo(unchanged: true, "pinvoke.o", "corebindings.o", "driver.o"); + + pathsDict.Remove(mainAssembly); + if (buildArgs.AOT) + { + // link optimization flag change affects .bc->.o files too, but + // it might result in only *some* files being *changed, + // so, don't check for those + // Link optimization flag is set to Compile optimization flag, if unset + // so, it affects .bc files too! + foreach (string key in pathsDict.Keys.ToArray()) + { + if (key.EndsWith(".dll.bc", StringComparison.Ordinal) || key.EndsWith(".dll.o", StringComparison.Ordinal)) + pathsDict.Remove(key); + } + } + + var originalStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + // Rebuild + + string output = Rebuild(nativeRelink: true, invariant: false, buildArgs, id, extraBuildArgs: $" {cflags} {ldflags}", verbosity: "normal"); + var newStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + CompareStat(originalStat, newStat, pathsDict.Values); + + string runOutput = RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); + AssertSubstring($"Found statically linked AOT module '{Path.GetFileNameWithoutExtension(mainAssembly)}'", runOutput, + contains: buildArgs.AOT); + } + + private (BuildArgs BuildArgs, BuildPaths paths) FirstNativeBuild(string programText, bool nativeRelink, bool invariant, BuildArgs buildArgs, string id, string extraProperties="") + { + buildArgs = GenerateProjectContents(buildArgs, nativeRelink, invariant, extraProperties); + BuildProject(buildArgs, + initProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText), + dotnetWasmFromRuntimePack: false, + hasIcudt: !invariant, + id: id, + createProject: true); + + RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: RunHost.V8, id: id); + return (buildArgs, GetBuildPaths(buildArgs)); + } + + private string Rebuild(bool nativeRelink, bool invariant, BuildArgs buildArgs, string id, string extraProperties="", string extraBuildArgs="", string? verbosity=null) + { + if (!_buildContext.TryGetBuildFor(buildArgs, out BuildProduct? product)) + throw new XunitException($"Test bug: could not get the build product in the cache"); + + File.Move(product!.LogFile, Path.ChangeExtension(product.LogFile!, ".first.binlog")); + + buildArgs = buildArgs with { ExtraBuildArgs = $"{buildArgs.ExtraBuildArgs} {extraBuildArgs}" }; + var newBuildArgs = GenerateProjectContents(buildArgs, nativeRelink, invariant, extraProperties); + + // key(buildArgs) being changed + _buildContext.RemoveFromCache(product.ProjectDir); + _buildContext.CacheBuild(newBuildArgs, product); + + if (buildArgs.ProjectFileContents != newBuildArgs.ProjectFileContents) + File.WriteAllText(Path.Combine(_projectDir!, $"{buildArgs.ProjectName}.csproj"), buildArgs.ProjectFileContents); + buildArgs = newBuildArgs; + + _testOutput.WriteLine($"{Environment.NewLine}Rebuilding with no changes ..{Environment.NewLine}"); + (_, string output) = BuildProject(buildArgs, + id: id, + dotnetWasmFromRuntimePack: false, + hasIcudt: !invariant, + createProject: false, + useCache: false, + verbosity: verbosity); + + return output; + } + + private BuildArgs GenerateProjectContents(BuildArgs buildArgs, bool nativeRelink, bool invariant, string extraProperties) + { + StringBuilder propertiesBuilder = new(); + propertiesBuilder.Append("<_WasmDevel>true"); + if (nativeRelink) + propertiesBuilder.Append($"true"); + if (invariant) + propertiesBuilder.Append($"true"); + propertiesBuilder.Append(extraProperties); + + return ExpandBuildArgs(buildArgs, propertiesBuilder.ToString()); + } + + private void CompareStat(IDictionary oldStat, IDictionary newStat, IEnumerable<(string fullpath, bool unchanged)> expected) + { + StringBuilder msg = new(); + foreach (var expect in expected) + { + string expectFilename = Path.GetFileName(expect.fullpath); + if (!oldStat.TryGetValue(expectFilename, out FileStat? oldFs)) + { + msg.AppendLine($"Could not find an entry for {expectFilename} in old files"); + continue; + } + + if (!newStat.TryGetValue(expectFilename, out FileStat? newFs)) + { + msg.AppendLine($"Could not find an entry for {expectFilename} in new files"); + continue; + } + + bool actualUnchanged = oldFs == newFs; + if (expect.unchanged && !actualUnchanged) + { + msg.AppendLine($"[Expected unchanged file: {expectFilename}]{Environment.NewLine}" + + $" old: {oldFs}{Environment.NewLine}" + + $" new: {newFs}"); + } + else if (!expect.unchanged && actualUnchanged) + { + msg.AppendLine($"[Expected changed file: {expectFilename}]{Environment.NewLine}" + + $" {newFs}"); + } + } + + if (msg.Length > 0) + throw new XunitException($"CompareStat failed:{Environment.NewLine}{msg}"); + } + + private IDictionary StatFiles(IEnumerable fullpaths) + { + Dictionary table = new(); + foreach (string file in fullpaths) + { + if (File.Exists(file)) + table.Add(Path.GetFileName(file), new FileStat(FullPath: file, Exists: true, LastWriteTimeUtc: File.GetLastWriteTimeUtc(file), Length: new FileInfo(file).Length)); + else + table.Add(Path.GetFileName(file), new FileStat(FullPath: file, Exists: false, LastWriteTimeUtc: DateTime.MinValue, Length: 0)); + } + + return table; + } + + private BuildPaths GetBuildPaths(BuildArgs buildArgs) + { + string objDir = GetObjDir(buildArgs.Config); + string bundleDir = Path.Combine(GetBinDir(baseDir: _projectDir, config: buildArgs.Config), "AppBundle"); + string wasmDir = Path.Combine(objDir, "wasm"); + + return new BuildPaths(wasmDir, objDir, GetBinDir(buildArgs.Config), bundleDir); + } + + private IDictionary GetFilesTable(BuildArgs buildArgs, BuildPaths paths, bool unchanged) + { + List files = new() + { + Path.Combine(paths.BinDir, "publish", $"{buildArgs.ProjectName}.dll"), + Path.Combine(paths.ObjWasmDir, "driver.o"), + Path.Combine(paths.ObjWasmDir, "corebindings.o"), + Path.Combine(paths.ObjWasmDir, "pinvoke.o"), + + Path.Combine(paths.ObjWasmDir, "icall-table.h"), + Path.Combine(paths.ObjWasmDir, "pinvoke-table.h"), + Path.Combine(paths.ObjWasmDir, "driver-gen.c"), + + Path.Combine(paths.BundleDir, "dotnet.wasm"), + Path.Combine(paths.BundleDir, "dotnet.js") + }; + + if (buildArgs.AOT) + { + files.AddRange(new[] + { + Path.Combine(paths.ObjWasmDir, $"{buildArgs.ProjectName}.dll.bc"), + Path.Combine(paths.ObjWasmDir, $"{buildArgs.ProjectName}.dll.o"), + + Path.Combine(paths.ObjWasmDir, "System.Private.CoreLib.dll.bc"), + Path.Combine(paths.ObjWasmDir, "System.Private.CoreLib.dll.o"), + }); + } + + var dict = new Dictionary(); + foreach (var file in files) + dict[Path.GetFileName(file)] = (file, unchanged); + + return dict; + } + + private void AssertSubstring(string substring, string full, bool contains) + { + if (contains) + Assert.Contains(substring, full); + else + Assert.DoesNotContain(substring, full); + } + } + + internal record FileStat (bool Exists, DateTime LastWriteTimeUtc, long Length, string FullPath); + internal record BuildPaths(string ObjWasmDir, string ObjDir, string BinDir, string BundleDir); +} diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/RebuildTests.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/RebuildTests.cs index 769d22fa660dc9..e5ea83805522be 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/RebuildTests.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/RebuildTests.cs @@ -18,21 +18,23 @@ public RebuildTests(ITestOutputHelper output, SharedBuildPerTestClassFixture bui { } + public IEnumerable NonNativeDebugRebuildData() + => ConfigWithAOTData(aot: false, config: "Debug") + .WithRunHosts(RunHost.V8) + .UnwrapItemsAsArrays().ToList(); + [Theory] - [BuildAndRun(host: RunHost.V8, aot: false, parameters: false)] - [BuildAndRun(host: RunHost.V8, aot: false, parameters: true)] - [BuildAndRun(host: RunHost.V8, aot: true, parameters: false)] - public void NoOpRebuild(BuildArgs buildArgs, bool nativeRelink, RunHost host, string id) + [MemberData(nameof(NonNativeDebugRebuildData))] + public void NoOpRebuild(BuildArgs buildArgs, RunHost host, string id) { string projectName = $"rebuild_{buildArgs.Config}_{buildArgs.AOT}"; - bool dotnetWasmFromRuntimePack = !nativeRelink && !buildArgs.AOT; buildArgs = buildArgs with { ProjectName = projectName }; - buildArgs = ExpandBuildArgs(buildArgs, $"{(nativeRelink ? "true" : "false")}"); + buildArgs = ExpandBuildArgs(buildArgs); BuildProject(buildArgs, initProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), - dotnetWasmFromRuntimePack: dotnetWasmFromRuntimePack, + dotnetWasmFromRuntimePack: true, id: id, createProject: true); @@ -48,7 +50,7 @@ public void NoOpRebuild(BuildArgs buildArgs, bool nativeRelink, RunHost host, st // no-op Rebuild BuildProject(buildArgs, id: id, - dotnetWasmFromRuntimePack: dotnetWasmFromRuntimePack, + dotnetWasmFromRuntimePack: true, createProject: false, useCache: false); diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/SharedBuildPerTestClassFixture.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/SharedBuildPerTestClassFixture.cs index e84a151bd5b02d..03437e67302133 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/SharedBuildPerTestClassFixture.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/SharedBuildPerTestClassFixture.cs @@ -18,14 +18,15 @@ public class SharedBuildPerTestClassFixture : IDisposable public void CacheBuild(BuildArgs buildArgs, BuildProduct product) => _buildPaths.Add(buildArgs, product); - public void RemoveFromCache(string buildPath) + public void RemoveFromCache(string buildPath, bool keepDir=true) { KeyValuePair? foundKvp = _buildPaths.Where(kvp => kvp.Value.ProjectDir == buildPath).SingleOrDefault(); if (foundKvp == null) throw new Exception($"Could not find build path {buildPath} in cache to remove."); _buildPaths.Remove(foundKvp.Value.Key); - RemoveDirectory(buildPath); + if (!keepDir) + RemoveDirectory(buildPath); } public bool TryGetBuildFor(BuildArgs buildArgs, [NotNullWhen(true)] out BuildProduct? product) @@ -42,6 +43,9 @@ public void Dispose() private void RemoveDirectory(string path) { + if (EnvironmentVariables.SkipProjectCleanup == "1") + return; + try { Directory.Delete(path, recursive: true); From 3658e9e84466cc6eb3c811b07613c1289a1210d1 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Tue, 10 Aug 2021 04:12:05 -0400 Subject: [PATCH 05/23] Disable non-wasm builds, for testing --- eng/pipelines/common/platform-matrix.yml | 635 ----------------------- 1 file changed, 635 deletions(-) diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index 15fe948e92967e..6dab0ab529fe2a 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -26,185 +26,6 @@ parameters: jobs: -# Linux arm -- ${{ if or(containsValue(parameters.platforms, 'Linux_arm'), in(parameters.platformGroup, 'all', 'gcstress')) }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: Linux - archType: arm - targetRid: linux-arm - platform: Linux_arm - container: - image: ubuntu-16.04-cross-20210719121212-8a8d3be - registry: mcr - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - crossBuild: true - crossrootfsDir: '/crossrootfs/arm' - ${{ insert }}: ${{ parameters.jobParameters }} - -# Linux arm64 - -- ${{ if or(containsValue(parameters.platforms, 'Linux_arm64'), in(parameters.platformGroup, 'all', 'gcstress')) }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: Linux - archType: arm64 - targetRid: linux-arm64 - platform: Linux_arm64 - container: - image: ubuntu-16.04-cross-arm64-20210719121212-8a8d3be - registry: mcr - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - crossBuild: true - crossrootfsDir: '/crossrootfs/arm64' - ${{ insert }}: ${{ parameters.jobParameters }} - -# Linux musl x64 - -- ${{ if or(containsValue(parameters.platforms, 'Linux_musl_x64'), eq(parameters.platformGroup, 'all')) }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: Linux - osSubgroup: _musl - archType: x64 - targetRid: linux-musl-x64 - platform: Linux_musl_x64 - container: - image: alpine-3.9-WithNode-20210714125437-9b5bbc2 - registry: mcr - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Linux musl arm - -- ${{ if or(containsValue(parameters.platforms, 'Linux_musl_arm'), eq(parameters.platformGroup, 'all')) }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: Linux - osSubgroup: _musl - archType: arm - targetRid: linux-musl-arm - platform: Linux_musl_arm - container: - image: ubuntu-16.04-cross-arm-alpine-20210719121212-044d5b9 - registry: mcr - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - crossBuild: true - crossrootfsDir: '/crossrootfs/arm' - ${{ insert }}: ${{ parameters.jobParameters }} - -# Linux musl arm64 - -- ${{ if or(containsValue(parameters.platforms, 'Linux_musl_arm64'), eq(parameters.platformGroup, 'all')) }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: Linux - osSubgroup: _musl - archType: arm64 - targetRid: linux-musl-arm64 - platform: Linux_musl_arm64 - container: - image: ubuntu-16.04-cross-arm64-alpine-20210719121212-b2c2436 - registry: mcr - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - crossBuild: true - crossrootfsDir: '/crossrootfs/arm64' - ${{ insert }}: ${{ parameters.jobParameters }} - -# Linux x64 - -- ${{ if or(containsValue(parameters.platforms, 'Linux_x64'), containsValue(parameters.platforms, 'CoreClrTestBuildHost'), in(parameters.platformGroup, 'all', 'gcstress')) }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: Linux - archType: x64 - targetRid: linux-x64 - platform: Linux_x64 - container: - image: centos-7-20210714125435-9b5bbc2 - registry: mcr - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Linux x64 Source Build - -- ${{ if containsValue(parameters.platforms, 'SourceBuild_Linux_x64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: Linux - archType: x64 - targetRid: linux-x64 - platform: Linux_x64 - container: - image: centos-7-source-build-20210714125450-5d87b80 - registry: mcr - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - buildingOnSourceBuildImage: true - # WebAssembly - ${{ if containsValue(parameters.platforms, 'Browser_wasm') }}: @@ -250,459 +71,3 @@ jobs: platforms: ${{ parameters.platforms }} ${{ insert }}: ${{ parameters.jobParameters }} -# FreeBSD -- ${{ if containsValue(parameters.platforms, 'FreeBSD_x64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: FreeBSD - archType: x64 - targetRid: freebsd-x64 - platform: FreeBSD_x64 - container: - image: ubuntu-18.04-cross-freebsd-11-20200407092345-a84b0d2 - registry: mcr - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - buildConfig: ${{ parameters.buildConfig }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - crossBuild: true - crossrootfsDir: '/crossrootfs/x64' - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Android x64 - -- ${{ if containsValue(parameters.platforms, 'Android_x64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: Android - archType: x64 - targetRid: android-x64 - platform: Android_x64 - container: - image: ubuntu-18.04-android-20200422191843-e2c3f83 - registry: mcr - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Android x86 - -- ${{ if containsValue(parameters.platforms, 'Android_x86') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: Android - archType: x86 - targetRid: android-x86 - platform: Android_x86 - container: - image: ubuntu-18.04-android-20200422191843-e2c3f83 - registry: mcr - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Android arm - -- ${{ if containsValue(parameters.platforms, 'Android_arm') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: Android - archType: arm - targetRid: android-arm - platform: Android_arm - container: - image: ubuntu-18.04-android-20200422191843-e2c3f83 - registry: mcr - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Android arm64 - -- ${{ if containsValue(parameters.platforms, 'Android_arm64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: Android - archType: arm64 - targetRid: android-arm64 - platform: Android_arm64 - container: - image: ubuntu-18.04-android-20200422191843-e2c3f83 - registry: mcr - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Mac Catalyst x64 - -- ${{ if containsValue(parameters.platforms, 'MacCatalyst_x64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: MacCatalyst - archType: x64 - targetRid: maccatalyst-x64 - platform: MacCatalyst_x64 - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Mac Catalyst arm64 - -- ${{ if containsValue(parameters.platforms, 'MacCatalyst_arm64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: MacCatalyst - archType: arm64 - targetRid: maccatalyst-arm64 - platform: MacCatalyst_arm64 - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# tvOS arm64 - -- ${{ if containsValue(parameters.platforms, 'tvOS_arm64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: tvOS - archType: arm64 - targetRid: tvos-arm64 - platform: tvOS_arm64 - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# tvOS Simulator x64 - -- ${{ if containsValue(parameters.platforms, 'tvOSSimulator_x64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: tvOSSimulator - archType: x64 - targetRid: tvossimulator-x64 - platform: tvOSSimulator_x64 - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# tvOS Simulator arm64 - -- ${{ if containsValue(parameters.platforms, 'tvOSSimulator_arm64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: tvOSSimulator - archType: arm64 - targetRid: tvossimulator-arm64 - platform: tvOSSimulator_arm64 - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# iOS arm - -- ${{ if containsValue(parameters.platforms, 'iOS_arm') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: iOS - archType: arm - targetRid: ios-arm - platform: iOS_arm - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# iOS arm64 - -- ${{ if containsValue(parameters.platforms, 'iOS_arm64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: iOS - archType: arm64 - targetRid: ios-arm64 - platform: iOS_arm64 - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# iOS Simulator x64 - -- ${{ if containsValue(parameters.platforms, 'iOSSimulator_x64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: iOSSimulator - archType: x64 - targetRid: iossimulator-x64 - platform: iOSSimulator_x64 - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# iOS Simulator x86 - -- ${{ if containsValue(parameters.platforms, 'iOSSimulator_x86') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: iOSSimulator - archType: x86 - targetRid: iossimulator-x86 - platform: iOSsimulator_x86 - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - managedTestBuildOsGroup: OSX - ${{ insert }}: ${{ parameters.jobParameters }} - -# iOS Simulator arm64 - -- ${{ if containsValue(parameters.platforms, 'iOSSimulator_arm64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: iOSSimulator - archType: arm64 - targetRid: iossimulator-arm64 - platform: iOSSimulator_arm64 - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# macOS arm64 - -- ${{ if containsValue(parameters.platforms, 'OSX_arm64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: OSX - archType: arm64 - targetRid: osx-arm64 - platform: OSX_arm64 - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - crossBuild: true - ${{ insert }}: ${{ parameters.jobParameters }} - -# macOS x64 - -- ${{ if or(containsValue(parameters.platforms, 'OSX_x64'), eq(parameters.platformGroup, 'all')) }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: OSX - archType: x64 - targetRid: osx-x64 - platform: OSX_x64 - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Windows x64 - -- ${{ if or(containsValue(parameters.platforms, 'windows_x64'), in(parameters.platformGroup, 'all', 'gcstress')) }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: windows - archType: x64 - targetRid: win-x64 - platform: windows_x64 - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Windows x86 - -- ${{ if or(containsValue(parameters.platforms, 'windows_x86'), in(parameters.platformGroup, 'all', 'gcstress')) }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: windows - archType: x86 - targetRid: win-x86 - platform: windows_x86 - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Windows arm -- ${{ if or(containsValue(parameters.platforms, 'windows_arm'), eq(parameters.platformGroup, 'all')) }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: windows - archType: arm - targetRid: win-arm - platform: windows_arm - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Windows arm64 - -- ${{ if or(containsValue(parameters.platforms, 'windows_arm64'), in(parameters.platformGroup, 'all', 'gcstress')) }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: windows - archType: arm64 - targetRid: win-arm64 - platform: windows_arm64 - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} From 85046e669b4067d47a23944100c0fe3024344789 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Tue, 10 Aug 2021 15:26:39 -0400 Subject: [PATCH 06/23] Add miggins LogAsErrorException.cs --- src/tasks/Common/LogAsErrorException.cs | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 src/tasks/Common/LogAsErrorException.cs diff --git a/src/tasks/Common/LogAsErrorException.cs b/src/tasks/Common/LogAsErrorException.cs new file mode 100644 index 00000000000000..a976de8cbbc4e2 --- /dev/null +++ b/src/tasks/Common/LogAsErrorException.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +public class LogAsErrorException : System.Exception +{ + public LogAsErrorException(string message) : base(message) + { + } +} From af1c40a24b9945009dccb66d26781c4fd894ad7c Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Tue, 10 Aug 2021 15:28:01 -0400 Subject: [PATCH 07/23] Bump sdk for workload testing to 6.0.100-rc.1.21410.3 --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 3446baa3f981df..805c030204bf11 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -164,7 +164,7 @@ 2.0.4 4.12.0 2.14.3 - 6.0.100-rc.1.21370.2 + 6.0.100-rc.1.21410.3 5.0.0-preview-20201009.2 From a4bc1ebf92e7c83b32550b380408d7e64153afef Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Tue, 10 Aug 2021 16:24:54 -0400 Subject: [PATCH 08/23] fix build --- src/tests/BuildWasmApps/Wasm.Build.Tests/RebuildTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/RebuildTests.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/RebuildTests.cs index e5ea83805522be..0f43fbd3967f0d 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/RebuildTests.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/RebuildTests.cs @@ -2,7 +2,9 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Collections.Generic; using System.IO; +using System.Linq; using Xunit; using Xunit.Abstractions; using Xunit.Sdk; From c8b75fbcf22ff34f0439e9a088d1695900eb6655 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Tue, 10 Aug 2021 20:19:00 -0400 Subject: [PATCH 09/23] MonoAOTCompiler: use the full path to copy the final .bc file --- src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 36 +++++++++----------- src/tasks/WasmAppBuilder/EmccCompile.cs | 2 ++ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index ddb359609623bc..00ddf7b660dd63 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -207,19 +207,16 @@ public class MonoAOTCompiler : Microsoft.Build.Utilities.Task public override bool Execute() { - if (string.IsNullOrEmpty(CompilerBinaryPath)) - { - throw new ArgumentException($"'{nameof(CompilerBinaryPath)}' is required.", nameof(CompilerBinaryPath)); - } - if (!File.Exists(CompilerBinaryPath)) { - throw new ArgumentException($"'{CompilerBinaryPath}' doesn't exist.", nameof(CompilerBinaryPath)); + Log.LogError($"{nameof(CompilerBinaryPath)}='{CompilerBinaryPath}' doesn't exist."); + return false; } if (Assemblies.Length == 0) { - throw new ArgumentException($"'{nameof(Assemblies)}' is required.", nameof(Assemblies)); + Log.LogError($"'{nameof(Assemblies)}' is required."); + return false; } if (!Path.IsPathRooted(OutputDir)) @@ -661,31 +658,32 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) return false; } - if (HaveAsmDependenciesChanged(assembly, depfile, out string? cause)) + if (Path.GetExtension(tmpOutputFileName) == ".tmp") { - if (Path.GetExtension(tmpOutputFileName) == ".tmp") - { - string finalFileName = Path.GetFileNameWithoutExtension(tmpOutputFileName); + string finalPath = Path.Combine(Path.GetDirectoryName(tmpOutputFileName) ?? string.Empty, + Path.GetFileNameWithoutExtension(tmpOutputFileName)); - if (File.Exists(finalFileName)) - File.Delete(finalFileName); + if (HaveAsmDependenciesChanged(assembly, depfile, out string? cause)) + { + if (File.Exists(finalPath)) + File.Delete(finalPath); - File.Copy(tmpOutputFileName, finalFileName); + File.Copy(tmpOutputFileName, finalPath); - Log.LogMessage(MessageImportance.Low, $"[AOT] {assembly} because {cause}"); - _fileWrites.Add(finalFileName); + Log.LogMessage(MessageImportance.Low, $"Copying {tmpOutputFileName} to {finalPath} because {cause}"); + _fileWrites.Add(finalPath); int count = Interlocked.Increment(ref _numCompiled); - Log.LogMessage(MessageImportance.High, $"[{count}/{_totalNumAssemblies}] {assemblyFilename} -> {Path.GetFileName(finalFileName)}"); + Log.LogMessage(MessageImportance.High, $"[{count}/{_totalNumAssemblies}] {assemblyFilename} -> {Path.GetFileName(finalPath)}"); } else { - Log.LogWarning($"Bug: expected tmpOutputFileName to end in .tmp: {tmpOutputFileName}"); + Log.LogMessage(MessageImportance.Low, $"Skipping copying over {finalPath} as the contents are unchanged"); } } else { - Log.LogMessage(MessageImportance.Low, $"Skipping precompiling assembly {assembly} because it has unchanged hash."); + Log.LogWarning($"Bug: expected tmpOutputFileName to end in .tmp: {tmpOutputFileName}"); } File.Delete(responseFilePath); diff --git a/src/tasks/WasmAppBuilder/EmccCompile.cs b/src/tasks/WasmAppBuilder/EmccCompile.cs index 0c85b89c6ad2c0..55faa121d9db4b 100644 --- a/src/tasks/WasmAppBuilder/EmccCompile.cs +++ b/src/tasks/WasmAppBuilder/EmccCompile.cs @@ -184,6 +184,8 @@ bool ProcessSourceFile(string srcFile, string objFile) if (!Utils.CopyIfDifferent(tmpObjFile, objFile, useHash: true)) Log.LogMessage(MessageImportance.Low, $"Did not overwrite {objFile} as the contents are unchanged"); + else + Log.LogMessage(MessageImportance.Low, $"Copied {tmpObjFile} to {objFile}"); ITaskItem newItem = new TaskItem(objFile); newItem.SetMetadata("SourceFile", srcFile); From f9aac565958c055bfd4c3104ccac21c1d7eca2e3 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Tue, 10 Aug 2021 23:58:30 -0400 Subject: [PATCH 10/23] Make the method used with `MemberData`, static otherwise xunit just shows a cryptic: ``` Wasm.Build.Tests.RebuildTests.NoOpRebuild [STARTING] Wasm.Build.Tests.RebuildTests.NoOpRebuild [FAIL] System.NotSupportedException : Specified method is not supported. ``` --- src/tests/BuildWasmApps/Wasm.Build.Tests/RebuildTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/RebuildTests.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/RebuildTests.cs index 0f43fbd3967f0d..961c141348275d 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/RebuildTests.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/RebuildTests.cs @@ -20,7 +20,7 @@ public RebuildTests(ITestOutputHelper output, SharedBuildPerTestClassFixture bui { } - public IEnumerable NonNativeDebugRebuildData() + public static IEnumerable NonNativeDebugRebuildData() => ConfigWithAOTData(aot: false, config: "Debug") .WithRunHosts(RunHost.V8) .UnwrapItemsAsArrays().ToList(); From 1d031c04e13780ec73180ba6f06a37ee42c24203 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 11 Aug 2021 02:54:50 -0400 Subject: [PATCH 11/23] add back builds --- eng/pipelines/common/platform-matrix.yml | 635 +++++++++++++++++++++++ 1 file changed, 635 insertions(+) diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index 6dab0ab529fe2a..15fe948e92967e 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -26,6 +26,185 @@ parameters: jobs: +# Linux arm +- ${{ if or(containsValue(parameters.platforms, 'Linux_arm'), in(parameters.platformGroup, 'all', 'gcstress')) }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Linux + archType: arm + targetRid: linux-arm + platform: Linux_arm + container: + image: ubuntu-16.04-cross-20210719121212-8a8d3be + registry: mcr + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + crossBuild: true + crossrootfsDir: '/crossrootfs/arm' + ${{ insert }}: ${{ parameters.jobParameters }} + +# Linux arm64 + +- ${{ if or(containsValue(parameters.platforms, 'Linux_arm64'), in(parameters.platformGroup, 'all', 'gcstress')) }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Linux + archType: arm64 + targetRid: linux-arm64 + platform: Linux_arm64 + container: + image: ubuntu-16.04-cross-arm64-20210719121212-8a8d3be + registry: mcr + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + crossBuild: true + crossrootfsDir: '/crossrootfs/arm64' + ${{ insert }}: ${{ parameters.jobParameters }} + +# Linux musl x64 + +- ${{ if or(containsValue(parameters.platforms, 'Linux_musl_x64'), eq(parameters.platformGroup, 'all')) }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Linux + osSubgroup: _musl + archType: x64 + targetRid: linux-musl-x64 + platform: Linux_musl_x64 + container: + image: alpine-3.9-WithNode-20210714125437-9b5bbc2 + registry: mcr + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Linux musl arm + +- ${{ if or(containsValue(parameters.platforms, 'Linux_musl_arm'), eq(parameters.platformGroup, 'all')) }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Linux + osSubgroup: _musl + archType: arm + targetRid: linux-musl-arm + platform: Linux_musl_arm + container: + image: ubuntu-16.04-cross-arm-alpine-20210719121212-044d5b9 + registry: mcr + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + crossBuild: true + crossrootfsDir: '/crossrootfs/arm' + ${{ insert }}: ${{ parameters.jobParameters }} + +# Linux musl arm64 + +- ${{ if or(containsValue(parameters.platforms, 'Linux_musl_arm64'), eq(parameters.platformGroup, 'all')) }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Linux + osSubgroup: _musl + archType: arm64 + targetRid: linux-musl-arm64 + platform: Linux_musl_arm64 + container: + image: ubuntu-16.04-cross-arm64-alpine-20210719121212-b2c2436 + registry: mcr + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + crossBuild: true + crossrootfsDir: '/crossrootfs/arm64' + ${{ insert }}: ${{ parameters.jobParameters }} + +# Linux x64 + +- ${{ if or(containsValue(parameters.platforms, 'Linux_x64'), containsValue(parameters.platforms, 'CoreClrTestBuildHost'), in(parameters.platformGroup, 'all', 'gcstress')) }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Linux + archType: x64 + targetRid: linux-x64 + platform: Linux_x64 + container: + image: centos-7-20210714125435-9b5bbc2 + registry: mcr + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Linux x64 Source Build + +- ${{ if containsValue(parameters.platforms, 'SourceBuild_Linux_x64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Linux + archType: x64 + targetRid: linux-x64 + platform: Linux_x64 + container: + image: centos-7-source-build-20210714125450-5d87b80 + registry: mcr + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + buildingOnSourceBuildImage: true + # WebAssembly - ${{ if containsValue(parameters.platforms, 'Browser_wasm') }}: @@ -71,3 +250,459 @@ jobs: platforms: ${{ parameters.platforms }} ${{ insert }}: ${{ parameters.jobParameters }} +# FreeBSD +- ${{ if containsValue(parameters.platforms, 'FreeBSD_x64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: FreeBSD + archType: x64 + targetRid: freebsd-x64 + platform: FreeBSD_x64 + container: + image: ubuntu-18.04-cross-freebsd-11-20200407092345-a84b0d2 + registry: mcr + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + buildConfig: ${{ parameters.buildConfig }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + crossBuild: true + crossrootfsDir: '/crossrootfs/x64' + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Android x64 + +- ${{ if containsValue(parameters.platforms, 'Android_x64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Android + archType: x64 + targetRid: android-x64 + platform: Android_x64 + container: + image: ubuntu-18.04-android-20200422191843-e2c3f83 + registry: mcr + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Android x86 + +- ${{ if containsValue(parameters.platforms, 'Android_x86') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Android + archType: x86 + targetRid: android-x86 + platform: Android_x86 + container: + image: ubuntu-18.04-android-20200422191843-e2c3f83 + registry: mcr + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Android arm + +- ${{ if containsValue(parameters.platforms, 'Android_arm') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Android + archType: arm + targetRid: android-arm + platform: Android_arm + container: + image: ubuntu-18.04-android-20200422191843-e2c3f83 + registry: mcr + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Android arm64 + +- ${{ if containsValue(parameters.platforms, 'Android_arm64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Android + archType: arm64 + targetRid: android-arm64 + platform: Android_arm64 + container: + image: ubuntu-18.04-android-20200422191843-e2c3f83 + registry: mcr + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Mac Catalyst x64 + +- ${{ if containsValue(parameters.platforms, 'MacCatalyst_x64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: MacCatalyst + archType: x64 + targetRid: maccatalyst-x64 + platform: MacCatalyst_x64 + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Mac Catalyst arm64 + +- ${{ if containsValue(parameters.platforms, 'MacCatalyst_arm64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: MacCatalyst + archType: arm64 + targetRid: maccatalyst-arm64 + platform: MacCatalyst_arm64 + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# tvOS arm64 + +- ${{ if containsValue(parameters.platforms, 'tvOS_arm64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: tvOS + archType: arm64 + targetRid: tvos-arm64 + platform: tvOS_arm64 + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# tvOS Simulator x64 + +- ${{ if containsValue(parameters.platforms, 'tvOSSimulator_x64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: tvOSSimulator + archType: x64 + targetRid: tvossimulator-x64 + platform: tvOSSimulator_x64 + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# tvOS Simulator arm64 + +- ${{ if containsValue(parameters.platforms, 'tvOSSimulator_arm64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: tvOSSimulator + archType: arm64 + targetRid: tvossimulator-arm64 + platform: tvOSSimulator_arm64 + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# iOS arm + +- ${{ if containsValue(parameters.platforms, 'iOS_arm') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: iOS + archType: arm + targetRid: ios-arm + platform: iOS_arm + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# iOS arm64 + +- ${{ if containsValue(parameters.platforms, 'iOS_arm64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: iOS + archType: arm64 + targetRid: ios-arm64 + platform: iOS_arm64 + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# iOS Simulator x64 + +- ${{ if containsValue(parameters.platforms, 'iOSSimulator_x64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: iOSSimulator + archType: x64 + targetRid: iossimulator-x64 + platform: iOSSimulator_x64 + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# iOS Simulator x86 + +- ${{ if containsValue(parameters.platforms, 'iOSSimulator_x86') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: iOSSimulator + archType: x86 + targetRid: iossimulator-x86 + platform: iOSsimulator_x86 + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + managedTestBuildOsGroup: OSX + ${{ insert }}: ${{ parameters.jobParameters }} + +# iOS Simulator arm64 + +- ${{ if containsValue(parameters.platforms, 'iOSSimulator_arm64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: iOSSimulator + archType: arm64 + targetRid: iossimulator-arm64 + platform: iOSSimulator_arm64 + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# macOS arm64 + +- ${{ if containsValue(parameters.platforms, 'OSX_arm64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: OSX + archType: arm64 + targetRid: osx-arm64 + platform: OSX_arm64 + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + crossBuild: true + ${{ insert }}: ${{ parameters.jobParameters }} + +# macOS x64 + +- ${{ if or(containsValue(parameters.platforms, 'OSX_x64'), eq(parameters.platformGroup, 'all')) }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: OSX + archType: x64 + targetRid: osx-x64 + platform: OSX_x64 + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Windows x64 + +- ${{ if or(containsValue(parameters.platforms, 'windows_x64'), in(parameters.platformGroup, 'all', 'gcstress')) }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: windows + archType: x64 + targetRid: win-x64 + platform: windows_x64 + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Windows x86 + +- ${{ if or(containsValue(parameters.platforms, 'windows_x86'), in(parameters.platformGroup, 'all', 'gcstress')) }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: windows + archType: x86 + targetRid: win-x86 + platform: windows_x86 + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Windows arm +- ${{ if or(containsValue(parameters.platforms, 'windows_arm'), eq(parameters.platformGroup, 'all')) }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: windows + archType: arm + targetRid: win-arm + platform: windows_arm + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Windows arm64 + +- ${{ if or(containsValue(parameters.platforms, 'windows_arm64'), in(parameters.platformGroup, 'all', 'gcstress')) }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: windows + archType: arm64 + targetRid: win-arm64 + platform: windows_arm64 + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} From 41976d7394843bcca2a210445b587faa0568e077 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 11 Aug 2021 09:59:18 -0400 Subject: [PATCH 12/23] Split Wasm.Build.Tests into multiple helix jobs --- .../scenarios/BuildWasmAppsJobsList.txt | 11 ++++++++++ src/libraries/sendtohelixhelp.proj | 22 ++++++++++++++++++- .../Wasm.Build.Tests/Wasm.Build.Tests.csproj | 4 ++++ 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 eng/testing/scenarios/BuildWasmAppsJobsList.txt diff --git a/eng/testing/scenarios/BuildWasmAppsJobsList.txt b/eng/testing/scenarios/BuildWasmAppsJobsList.txt new file mode 100644 index 00000000000000..51c72c75db82d7 --- /dev/null +++ b/eng/testing/scenarios/BuildWasmAppsJobsList.txt @@ -0,0 +1,11 @@ +LocalEMSDKTests +RebuildTests +NativeRebuildTests +InvariantGlobalizationTests +NativeBuildTests +WasmBuildAppTest +BlazorWasmTests +WorkloadTests +SatelliteAssembliesTests +NativeLibraryTests +MainWithArgsTests diff --git a/src/libraries/sendtohelixhelp.proj b/src/libraries/sendtohelixhelp.proj index e9ddef9735eafb..ee8f200e42f49b 100644 --- a/src/libraries/sendtohelixhelp.proj +++ b/src/libraries/sendtohelixhelp.proj @@ -53,6 +53,8 @@ $(TestRunNamePrefix)$(TestRunNamePrefixSuffix)- $(TestRunNamePrefix)$(Scenario)- + $(RepositoryEngineeringDir)\testing\scenarios\BuildWasmAppsJobsList.txt + $(WaitForWorkItemCompletion) true $(RepoRoot)src\mono\wasm\emsdk\ @@ -378,6 +380,10 @@ + + + + <_WorkItem Include="$(TestArchiveRoot)browseronly/**/*.zip" Condition="'$(TargetOS)' == 'Browser' and '$(Scenario)' == 'WasmTestOnBrowser'" /> - + %(Identity) $(HelixCommand) $(_workItemTimeout) + + <_BuildWasmAppsPayloadArchive>@(_WorkItem) + + + + + $(_BuildWasmAppsPayloadArchive) + set "HELIX_XUNIT_ARGS=-class Wasm.Build.Tests.%(Identity)" + export "HELIX_XUNIT_ARGS=-class Wasm.Build.Tests.%(Identity)" + $(HelixCommand) + $(_workItemTimeout) + + + dotnet exec $XHARNESS_CLI_PATH $HELIX_WORKITEM_UPLOAD_ROOT/xharness-output diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/Wasm.Build.Tests.csproj b/src/tests/BuildWasmApps/Wasm.Build.Tests/Wasm.Build.Tests.csproj index d6709715ef0451..3d39f353c0779e 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/Wasm.Build.Tests.csproj +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/Wasm.Build.Tests.csproj @@ -60,6 +60,10 @@ dotnet exec xunit.console.dll $(AssemblyName).dll -xml %24XHARNESS_OUT/testResults.xml dotnet.exe exec xunit.console.dll $(AssemblyName).dll -xml %XHARNESS_OUT%\testResults.xml + + $(RunScriptCommand) %24HELIX_XUNIT_ARGS + $(RunScriptCommand) %HELIX_XUNIT_ARGS% + $(RunScriptCommand) -nocolor $(RunScriptCommand) -verbose From b008130a7886c2e2b9f16c83641c1b8c936082f6 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 11 Aug 2021 09:59:29 -0400 Subject: [PATCH 13/23] Revert "add back builds" This reverts commit 1d031c04e13780ec73180ba6f06a37ee42c24203. --- eng/pipelines/common/platform-matrix.yml | 635 ----------------------- 1 file changed, 635 deletions(-) diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index 15fe948e92967e..6dab0ab529fe2a 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -26,185 +26,6 @@ parameters: jobs: -# Linux arm -- ${{ if or(containsValue(parameters.platforms, 'Linux_arm'), in(parameters.platformGroup, 'all', 'gcstress')) }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: Linux - archType: arm - targetRid: linux-arm - platform: Linux_arm - container: - image: ubuntu-16.04-cross-20210719121212-8a8d3be - registry: mcr - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - crossBuild: true - crossrootfsDir: '/crossrootfs/arm' - ${{ insert }}: ${{ parameters.jobParameters }} - -# Linux arm64 - -- ${{ if or(containsValue(parameters.platforms, 'Linux_arm64'), in(parameters.platformGroup, 'all', 'gcstress')) }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: Linux - archType: arm64 - targetRid: linux-arm64 - platform: Linux_arm64 - container: - image: ubuntu-16.04-cross-arm64-20210719121212-8a8d3be - registry: mcr - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - crossBuild: true - crossrootfsDir: '/crossrootfs/arm64' - ${{ insert }}: ${{ parameters.jobParameters }} - -# Linux musl x64 - -- ${{ if or(containsValue(parameters.platforms, 'Linux_musl_x64'), eq(parameters.platformGroup, 'all')) }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: Linux - osSubgroup: _musl - archType: x64 - targetRid: linux-musl-x64 - platform: Linux_musl_x64 - container: - image: alpine-3.9-WithNode-20210714125437-9b5bbc2 - registry: mcr - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Linux musl arm - -- ${{ if or(containsValue(parameters.platforms, 'Linux_musl_arm'), eq(parameters.platformGroup, 'all')) }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: Linux - osSubgroup: _musl - archType: arm - targetRid: linux-musl-arm - platform: Linux_musl_arm - container: - image: ubuntu-16.04-cross-arm-alpine-20210719121212-044d5b9 - registry: mcr - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - crossBuild: true - crossrootfsDir: '/crossrootfs/arm' - ${{ insert }}: ${{ parameters.jobParameters }} - -# Linux musl arm64 - -- ${{ if or(containsValue(parameters.platforms, 'Linux_musl_arm64'), eq(parameters.platformGroup, 'all')) }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: Linux - osSubgroup: _musl - archType: arm64 - targetRid: linux-musl-arm64 - platform: Linux_musl_arm64 - container: - image: ubuntu-16.04-cross-arm64-alpine-20210719121212-b2c2436 - registry: mcr - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - crossBuild: true - crossrootfsDir: '/crossrootfs/arm64' - ${{ insert }}: ${{ parameters.jobParameters }} - -# Linux x64 - -- ${{ if or(containsValue(parameters.platforms, 'Linux_x64'), containsValue(parameters.platforms, 'CoreClrTestBuildHost'), in(parameters.platformGroup, 'all', 'gcstress')) }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: Linux - archType: x64 - targetRid: linux-x64 - platform: Linux_x64 - container: - image: centos-7-20210714125435-9b5bbc2 - registry: mcr - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Linux x64 Source Build - -- ${{ if containsValue(parameters.platforms, 'SourceBuild_Linux_x64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: Linux - archType: x64 - targetRid: linux-x64 - platform: Linux_x64 - container: - image: centos-7-source-build-20210714125450-5d87b80 - registry: mcr - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - buildingOnSourceBuildImage: true - # WebAssembly - ${{ if containsValue(parameters.platforms, 'Browser_wasm') }}: @@ -250,459 +71,3 @@ jobs: platforms: ${{ parameters.platforms }} ${{ insert }}: ${{ parameters.jobParameters }} -# FreeBSD -- ${{ if containsValue(parameters.platforms, 'FreeBSD_x64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: FreeBSD - archType: x64 - targetRid: freebsd-x64 - platform: FreeBSD_x64 - container: - image: ubuntu-18.04-cross-freebsd-11-20200407092345-a84b0d2 - registry: mcr - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - buildConfig: ${{ parameters.buildConfig }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - crossBuild: true - crossrootfsDir: '/crossrootfs/x64' - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Android x64 - -- ${{ if containsValue(parameters.platforms, 'Android_x64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: Android - archType: x64 - targetRid: android-x64 - platform: Android_x64 - container: - image: ubuntu-18.04-android-20200422191843-e2c3f83 - registry: mcr - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Android x86 - -- ${{ if containsValue(parameters.platforms, 'Android_x86') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: Android - archType: x86 - targetRid: android-x86 - platform: Android_x86 - container: - image: ubuntu-18.04-android-20200422191843-e2c3f83 - registry: mcr - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Android arm - -- ${{ if containsValue(parameters.platforms, 'Android_arm') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: Android - archType: arm - targetRid: android-arm - platform: Android_arm - container: - image: ubuntu-18.04-android-20200422191843-e2c3f83 - registry: mcr - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Android arm64 - -- ${{ if containsValue(parameters.platforms, 'Android_arm64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: Android - archType: arm64 - targetRid: android-arm64 - platform: Android_arm64 - container: - image: ubuntu-18.04-android-20200422191843-e2c3f83 - registry: mcr - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Mac Catalyst x64 - -- ${{ if containsValue(parameters.platforms, 'MacCatalyst_x64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: MacCatalyst - archType: x64 - targetRid: maccatalyst-x64 - platform: MacCatalyst_x64 - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Mac Catalyst arm64 - -- ${{ if containsValue(parameters.platforms, 'MacCatalyst_arm64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: MacCatalyst - archType: arm64 - targetRid: maccatalyst-arm64 - platform: MacCatalyst_arm64 - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# tvOS arm64 - -- ${{ if containsValue(parameters.platforms, 'tvOS_arm64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: tvOS - archType: arm64 - targetRid: tvos-arm64 - platform: tvOS_arm64 - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# tvOS Simulator x64 - -- ${{ if containsValue(parameters.platforms, 'tvOSSimulator_x64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: tvOSSimulator - archType: x64 - targetRid: tvossimulator-x64 - platform: tvOSSimulator_x64 - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# tvOS Simulator arm64 - -- ${{ if containsValue(parameters.platforms, 'tvOSSimulator_arm64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: tvOSSimulator - archType: arm64 - targetRid: tvossimulator-arm64 - platform: tvOSSimulator_arm64 - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# iOS arm - -- ${{ if containsValue(parameters.platforms, 'iOS_arm') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: iOS - archType: arm - targetRid: ios-arm - platform: iOS_arm - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# iOS arm64 - -- ${{ if containsValue(parameters.platforms, 'iOS_arm64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: iOS - archType: arm64 - targetRid: ios-arm64 - platform: iOS_arm64 - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# iOS Simulator x64 - -- ${{ if containsValue(parameters.platforms, 'iOSSimulator_x64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: iOSSimulator - archType: x64 - targetRid: iossimulator-x64 - platform: iOSSimulator_x64 - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# iOS Simulator x86 - -- ${{ if containsValue(parameters.platforms, 'iOSSimulator_x86') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: iOSSimulator - archType: x86 - targetRid: iossimulator-x86 - platform: iOSsimulator_x86 - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - managedTestBuildOsGroup: OSX - ${{ insert }}: ${{ parameters.jobParameters }} - -# iOS Simulator arm64 - -- ${{ if containsValue(parameters.platforms, 'iOSSimulator_arm64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: iOSSimulator - archType: arm64 - targetRid: iossimulator-arm64 - platform: iOSSimulator_arm64 - jobParameters: - runtimeFlavor: mono - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# macOS arm64 - -- ${{ if containsValue(parameters.platforms, 'OSX_arm64') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: OSX - archType: arm64 - targetRid: osx-arm64 - platform: OSX_arm64 - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - crossBuild: true - ${{ insert }}: ${{ parameters.jobParameters }} - -# macOS x64 - -- ${{ if or(containsValue(parameters.platforms, 'OSX_x64'), eq(parameters.platformGroup, 'all')) }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: OSX - archType: x64 - targetRid: osx-x64 - platform: OSX_x64 - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Windows x64 - -- ${{ if or(containsValue(parameters.platforms, 'windows_x64'), in(parameters.platformGroup, 'all', 'gcstress')) }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: windows - archType: x64 - targetRid: win-x64 - platform: windows_x64 - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Windows x86 - -- ${{ if or(containsValue(parameters.platforms, 'windows_x86'), in(parameters.platformGroup, 'all', 'gcstress')) }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: windows - archType: x86 - targetRid: win-x86 - platform: windows_x86 - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Windows arm -- ${{ if or(containsValue(parameters.platforms, 'windows_arm'), eq(parameters.platformGroup, 'all')) }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: windows - archType: arm - targetRid: win-arm - platform: windows_arm - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - -# Windows arm64 - -- ${{ if or(containsValue(parameters.platforms, 'windows_arm64'), in(parameters.platformGroup, 'all', 'gcstress')) }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: windows - archType: arm64 - targetRid: win-arm64 - platform: windows_arm64 - jobParameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} - stagedBuild: ${{ parameters.stagedBuild }} - buildConfig: ${{ parameters.buildConfig }} - ${{ if eq(parameters.passPlatforms, true) }}: - platforms: ${{ parameters.platforms }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} From 088a16572859ee002145a9749fb667071ae886de Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 11 Aug 2021 10:33:57 -0400 Subject: [PATCH 14/23] Split up native rebuild tests --- .../scenarios/BuildWasmAppsJobsList.txt | 24 +- .../Wasm.Build.Tests/NativeRebuildTests.cs | 394 ------------------ .../FlagsChangeRebuildTest.cs | 118 ++++++ .../NativeRebuildTestsBase.cs | 209 ++++++++++ .../NoopNativeRebuildTest.cs | 36 ++ .../ReferenceNewAssemblyRebuildTest.cs | 57 +++ .../SimpleSourceChangeRebuildTest.cs | 55 +++ 7 files changed, 492 insertions(+), 401 deletions(-) delete mode 100644 src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests.cs create mode 100644 src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/FlagsChangeRebuildTest.cs create mode 100644 src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NativeRebuildTestsBase.cs create mode 100644 src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NoopNativeRebuildTest.cs create mode 100644 src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/ReferenceNewAssemblyRebuildTest.cs create mode 100644 src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/SimpleSourceChangeRebuildTest.cs diff --git a/eng/testing/scenarios/BuildWasmAppsJobsList.txt b/eng/testing/scenarios/BuildWasmAppsJobsList.txt index 51c72c75db82d7..ae5981223de3de 100644 --- a/eng/testing/scenarios/BuildWasmAppsJobsList.txt +++ b/eng/testing/scenarios/BuildWasmAppsJobsList.txt @@ -1,11 +1,21 @@ -LocalEMSDKTests -RebuildTests -NativeRebuildTests +BlazorWasmTests +BuildAndRunAttribute +BuildEnvironment +BuildTestBase +DotNetCommand +FlagsChangeRebuildTest InvariantGlobalizationTests +LocalEMSDKTests +MainWithArgsTests NativeBuildTests +NativeLibraryTests +NativeRebuildTestsBase +NoopNativeRebuildTest +RebuildTests +ReferenceNewAssemblyRebuildTest +SatelliteAssembliesTests +SharedBuildPerTestClassFixture +SimpleSourceChangeRebuildTest +ToolCommand WasmBuildAppTest -BlazorWasmTests WorkloadTests -SatelliteAssembliesTests -NativeLibraryTests -MainWithArgsTests diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests.cs deleted file mode 100644 index b20935abe2f48d..00000000000000 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests.cs +++ /dev/null @@ -1,394 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.IO; -using System.Collections.Generic; -using System.Linq; -using Xunit; -using Xunit.Abstractions; -using Xunit.Sdk; -using System.Text; - -#nullable enable - -namespace Wasm.Build.Tests -{ - // TODO: test for runtime components - public class NativeRebuildTests : BuildTestBase - { - public NativeRebuildTests(ITestOutputHelper output, SharedBuildPerTestClassFixture buildContext) - : base(output, buildContext) - { - _enablePerTestCleanup = true; - } - - public static IEnumerable NativeBuildData() - { - List data = new(); - // relinking - data.AddRange(GetData(aot: false, nativeRelinking: true, invariant: false)); - data.AddRange(GetData(aot: false, nativeRelinking: true, invariant: true)); - - // aot - data.AddRange(GetData(aot: true, nativeRelinking: false, invariant: false)); - data.AddRange(GetData(aot: true, nativeRelinking: false, invariant: true)); - - return data; - - IEnumerable GetData(bool aot, bool nativeRelinking, bool invariant) - => ConfigWithAOTData(aot) - .Multiply(new object[] { nativeRelinking, invariant }) - .WithRunHosts(RunHost.V8) - .UnwrapItemsAsArrays().ToList().Dump(); - } - - [Theory] - [MemberData(nameof(NativeBuildData))] - public void NoOpRebuildForNativeBuilds(BuildArgs buildArgs, bool nativeRelink, bool invariant, RunHost host, string id) - { - buildArgs = buildArgs with { ProjectName = $"rebuild_noop_{buildArgs.Config}" }; - (buildArgs, BuildPaths paths) = FirstNativeBuild(s_mainReturns42, nativeRelink: nativeRelink, invariant: invariant, buildArgs, id); - - var pathsDict = GetFilesTable(buildArgs, paths, unchanged: true); - var originalStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); - - Rebuild(nativeRelink, invariant, buildArgs, id); - var newStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); - - CompareStat(originalStat, newStat, pathsDict.Values); - RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); - } - - [Theory] - [MemberData(nameof(NativeBuildData))] - public void SimpleStringChangeInSource(BuildArgs buildArgs, bool nativeRelink, bool invariant, RunHost host, string id) - { - buildArgs = buildArgs with { ProjectName = $"rebuild_simple_{buildArgs.Config}" }; - (buildArgs, BuildPaths paths) = FirstNativeBuild(s_mainReturns42, nativeRelink, invariant: invariant, buildArgs, id); - - string mainAssembly = $"{buildArgs.ProjectName}.dll"; - var pathsDict = GetFilesTable(buildArgs, paths, unchanged: true); - pathsDict.UpdateTo(unchanged: false, mainAssembly); - pathsDict.UpdateTo(unchanged: !buildArgs.AOT, "dotnet.wasm", "dotnet.js"); - - if (buildArgs.AOT) - pathsDict.UpdateTo(unchanged: false, $"{mainAssembly}.bc", $"{mainAssembly}.o"); - - var originalStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); - - // Changes - string mainResults55 = @" - public class TestClass { - public static int Main() - { - return 55; - } - }"; - File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), mainResults55); - - // Rebuild - Rebuild(nativeRelink, invariant, buildArgs, id); - var newStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); - - CompareStat(originalStat, newStat, pathsDict.Values); - RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 55, host: host, id: id); - } - - [Theory] - [MemberData(nameof(NativeBuildData))] - public void ReferenceNewAssembly(BuildArgs buildArgs, bool nativeRelink, bool invariant, RunHost host, string id) - { - buildArgs = buildArgs with { ProjectName = $"rebuild_tasks_{buildArgs.Config}" }; - (buildArgs, BuildPaths paths) = FirstNativeBuild(s_mainReturns42, nativeRelink, invariant: invariant, buildArgs, id); - - var pathsDict = GetFilesTable(buildArgs, paths, unchanged: false); - pathsDict.UpdateTo(unchanged: true, "corebindings.o"); - if (!buildArgs.AOT) // relinking - pathsDict.UpdateTo(unchanged: true, "driver-gen.c"); - - var originalStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); - - string programText = - @$" - using System; - using System.Text.Json; - public class Test - {{ - public static int Main() - {{" + - @" string json = ""{ \""name\"": \""value\"" }"";" + - @" var jdoc = JsonDocument.Parse($""{json}"", new JsonDocumentOptions());" + - @$" Console.WriteLine($""json: {{jdoc}}""); - return 42; - }} - }}"; - File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText); - - Rebuild(nativeRelink, invariant, buildArgs, id); - var newStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); - - CompareStat(originalStat, newStat, pathsDict.Values); - RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); - } - - public static IEnumerable FlagsChangesForNativeRelinkingData(bool aot) - => ConfigWithAOTData(aot, config: "Release").Multiply( - new object[] { /*cflags*/ "/p:EmccExtraCFlags=-g", /*ldflags*/ "" }, - new object[] { /*cflags*/ "", /*ldflags*/ "/p:EmccExtraLDFlags=-g" }, - new object[] { /*cflags*/ "/p:EmccExtraCFlags=-g", /*ldflags*/ "/p:EmccExtraLDFlags=-g" } - ).WithRunHosts(RunHost.V8).UnwrapItemsAsArrays().Dump(); - - [Theory] - [MemberData(nameof(FlagsChangesForNativeRelinkingData), parameters: /*aot*/ false)] - [MemberData(nameof(FlagsChangesForNativeRelinkingData), parameters: /*aot*/ true)] - public void ExtraEmccFlagsSetButNoRealChange(BuildArgs buildArgs, string extraCFlags, string extraLDFlags, RunHost host, string id) - { - buildArgs = buildArgs with { ProjectName = $"rebuild_flags_{buildArgs.Config}" }; - (buildArgs, BuildPaths paths) = FirstNativeBuild(s_mainReturns42, nativeRelink: true, invariant: false, buildArgs, id); - var pathsDict = GetFilesTable(buildArgs, paths, unchanged: true); - if (extraLDFlags.Length > 0) - pathsDict.UpdateTo(unchanged: false, "dotnet.wasm", "dotnet.js"); - - var originalStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); - - // Rebuild - - string mainAssembly = $"{buildArgs.ProjectName}.dll"; - string extraBuildArgs = $" {extraCFlags} {extraLDFlags}"; - string output = Rebuild(nativeRelink: true, invariant: false, buildArgs, id, extraBuildArgs: extraBuildArgs, verbosity: "normal"); - - var newStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); - CompareStat(originalStat, newStat, pathsDict.Values); - - // cflags: pinvoke get's compiled, but doesn't overwrite pinvoke.o - // and thus doesn't cause relinking - AssertSubstring("pinvoke.c -> pinvoke.o", output, contains: extraCFlags.Length > 0); - - // ldflags: link step args change, so it should trigger relink - AssertSubstring("wasm-opt", output, contains: extraLDFlags.Length > 0); - - if (buildArgs.AOT) - { - // ExtraEmccLDFlags does not affect .bc files - Assert.DoesNotContain("Compiling assembly bitcode files", output); - } - - string runOutput = RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); - AssertSubstring($"Found statically linked AOT module '{Path.GetFileNameWithoutExtension(mainAssembly)}'", runOutput, - contains: buildArgs.AOT); - } - - public static IEnumerable FlagsOnlyChangeData(bool aot) - => ConfigWithAOTData(aot, config: "Release").Multiply( - new object[] { /*cflags*/ "/p:EmccCompileOptimizationFlag=-O1", /*ldflags*/ "" }, - new object[] { /*cflags*/ "", /*ldflags*/ "/p:EmccLinkOptimizationFlag=-O0" } - ).WithRunHosts(RunHost.V8).UnwrapItemsAsArrays().Dump(); - - [Theory] - [MemberData(nameof(FlagsOnlyChangeData), parameters: /*aot*/ false)] - [MemberData(nameof(FlagsOnlyChangeData), parameters: /*aot*/ true)] - public void OptimizationFlagChange(BuildArgs buildArgs, string cflags, string ldflags, RunHost host, string id) - { - // force _WasmDevel=false, so we don't get -O0 - buildArgs = buildArgs with { ProjectName = $"rebuild_flags_{buildArgs.Config}", ExtraBuildArgs = "/p:_WasmDevel=false" }; - (buildArgs, BuildPaths paths) = FirstNativeBuild(s_mainReturns42, nativeRelink: true, invariant: false, buildArgs, id); - - string mainAssembly = $"{buildArgs.ProjectName}.dll"; - var pathsDict = GetFilesTable(buildArgs, paths, unchanged: false); - pathsDict.UpdateTo(unchanged: true, mainAssembly, "icall-table.h", "pinvoke-table.h", "driver-gen.c"); - if (cflags.Length == 0) - pathsDict.UpdateTo(unchanged: true, "pinvoke.o", "corebindings.o", "driver.o"); - - pathsDict.Remove(mainAssembly); - if (buildArgs.AOT) - { - // link optimization flag change affects .bc->.o files too, but - // it might result in only *some* files being *changed, - // so, don't check for those - // Link optimization flag is set to Compile optimization flag, if unset - // so, it affects .bc files too! - foreach (string key in pathsDict.Keys.ToArray()) - { - if (key.EndsWith(".dll.bc", StringComparison.Ordinal) || key.EndsWith(".dll.o", StringComparison.Ordinal)) - pathsDict.Remove(key); - } - } - - var originalStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); - - // Rebuild - - string output = Rebuild(nativeRelink: true, invariant: false, buildArgs, id, extraBuildArgs: $" {cflags} {ldflags}", verbosity: "normal"); - var newStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); - CompareStat(originalStat, newStat, pathsDict.Values); - - string runOutput = RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); - AssertSubstring($"Found statically linked AOT module '{Path.GetFileNameWithoutExtension(mainAssembly)}'", runOutput, - contains: buildArgs.AOT); - } - - private (BuildArgs BuildArgs, BuildPaths paths) FirstNativeBuild(string programText, bool nativeRelink, bool invariant, BuildArgs buildArgs, string id, string extraProperties="") - { - buildArgs = GenerateProjectContents(buildArgs, nativeRelink, invariant, extraProperties); - BuildProject(buildArgs, - initProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText), - dotnetWasmFromRuntimePack: false, - hasIcudt: !invariant, - id: id, - createProject: true); - - RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: RunHost.V8, id: id); - return (buildArgs, GetBuildPaths(buildArgs)); - } - - private string Rebuild(bool nativeRelink, bool invariant, BuildArgs buildArgs, string id, string extraProperties="", string extraBuildArgs="", string? verbosity=null) - { - if (!_buildContext.TryGetBuildFor(buildArgs, out BuildProduct? product)) - throw new XunitException($"Test bug: could not get the build product in the cache"); - - File.Move(product!.LogFile, Path.ChangeExtension(product.LogFile!, ".first.binlog")); - - buildArgs = buildArgs with { ExtraBuildArgs = $"{buildArgs.ExtraBuildArgs} {extraBuildArgs}" }; - var newBuildArgs = GenerateProjectContents(buildArgs, nativeRelink, invariant, extraProperties); - - // key(buildArgs) being changed - _buildContext.RemoveFromCache(product.ProjectDir); - _buildContext.CacheBuild(newBuildArgs, product); - - if (buildArgs.ProjectFileContents != newBuildArgs.ProjectFileContents) - File.WriteAllText(Path.Combine(_projectDir!, $"{buildArgs.ProjectName}.csproj"), buildArgs.ProjectFileContents); - buildArgs = newBuildArgs; - - _testOutput.WriteLine($"{Environment.NewLine}Rebuilding with no changes ..{Environment.NewLine}"); - (_, string output) = BuildProject(buildArgs, - id: id, - dotnetWasmFromRuntimePack: false, - hasIcudt: !invariant, - createProject: false, - useCache: false, - verbosity: verbosity); - - return output; - } - - private BuildArgs GenerateProjectContents(BuildArgs buildArgs, bool nativeRelink, bool invariant, string extraProperties) - { - StringBuilder propertiesBuilder = new(); - propertiesBuilder.Append("<_WasmDevel>true"); - if (nativeRelink) - propertiesBuilder.Append($"true"); - if (invariant) - propertiesBuilder.Append($"true"); - propertiesBuilder.Append(extraProperties); - - return ExpandBuildArgs(buildArgs, propertiesBuilder.ToString()); - } - - private void CompareStat(IDictionary oldStat, IDictionary newStat, IEnumerable<(string fullpath, bool unchanged)> expected) - { - StringBuilder msg = new(); - foreach (var expect in expected) - { - string expectFilename = Path.GetFileName(expect.fullpath); - if (!oldStat.TryGetValue(expectFilename, out FileStat? oldFs)) - { - msg.AppendLine($"Could not find an entry for {expectFilename} in old files"); - continue; - } - - if (!newStat.TryGetValue(expectFilename, out FileStat? newFs)) - { - msg.AppendLine($"Could not find an entry for {expectFilename} in new files"); - continue; - } - - bool actualUnchanged = oldFs == newFs; - if (expect.unchanged && !actualUnchanged) - { - msg.AppendLine($"[Expected unchanged file: {expectFilename}]{Environment.NewLine}" + - $" old: {oldFs}{Environment.NewLine}" + - $" new: {newFs}"); - } - else if (!expect.unchanged && actualUnchanged) - { - msg.AppendLine($"[Expected changed file: {expectFilename}]{Environment.NewLine}" + - $" {newFs}"); - } - } - - if (msg.Length > 0) - throw new XunitException($"CompareStat failed:{Environment.NewLine}{msg}"); - } - - private IDictionary StatFiles(IEnumerable fullpaths) - { - Dictionary table = new(); - foreach (string file in fullpaths) - { - if (File.Exists(file)) - table.Add(Path.GetFileName(file), new FileStat(FullPath: file, Exists: true, LastWriteTimeUtc: File.GetLastWriteTimeUtc(file), Length: new FileInfo(file).Length)); - else - table.Add(Path.GetFileName(file), new FileStat(FullPath: file, Exists: false, LastWriteTimeUtc: DateTime.MinValue, Length: 0)); - } - - return table; - } - - private BuildPaths GetBuildPaths(BuildArgs buildArgs) - { - string objDir = GetObjDir(buildArgs.Config); - string bundleDir = Path.Combine(GetBinDir(baseDir: _projectDir, config: buildArgs.Config), "AppBundle"); - string wasmDir = Path.Combine(objDir, "wasm"); - - return new BuildPaths(wasmDir, objDir, GetBinDir(buildArgs.Config), bundleDir); - } - - private IDictionary GetFilesTable(BuildArgs buildArgs, BuildPaths paths, bool unchanged) - { - List files = new() - { - Path.Combine(paths.BinDir, "publish", $"{buildArgs.ProjectName}.dll"), - Path.Combine(paths.ObjWasmDir, "driver.o"), - Path.Combine(paths.ObjWasmDir, "corebindings.o"), - Path.Combine(paths.ObjWasmDir, "pinvoke.o"), - - Path.Combine(paths.ObjWasmDir, "icall-table.h"), - Path.Combine(paths.ObjWasmDir, "pinvoke-table.h"), - Path.Combine(paths.ObjWasmDir, "driver-gen.c"), - - Path.Combine(paths.BundleDir, "dotnet.wasm"), - Path.Combine(paths.BundleDir, "dotnet.js") - }; - - if (buildArgs.AOT) - { - files.AddRange(new[] - { - Path.Combine(paths.ObjWasmDir, $"{buildArgs.ProjectName}.dll.bc"), - Path.Combine(paths.ObjWasmDir, $"{buildArgs.ProjectName}.dll.o"), - - Path.Combine(paths.ObjWasmDir, "System.Private.CoreLib.dll.bc"), - Path.Combine(paths.ObjWasmDir, "System.Private.CoreLib.dll.o"), - }); - } - - var dict = new Dictionary(); - foreach (var file in files) - dict[Path.GetFileName(file)] = (file, unchanged); - - return dict; - } - - private void AssertSubstring(string substring, string full, bool contains) - { - if (contains) - Assert.Contains(substring, full); - else - Assert.DoesNotContain(substring, full); - } - } - - internal record FileStat (bool Exists, DateTime LastWriteTimeUtc, long Length, string FullPath); - internal record BuildPaths(string ObjWasmDir, string ObjDir, string BinDir, string BundleDir); -} diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/FlagsChangeRebuildTest.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/FlagsChangeRebuildTest.cs new file mode 100644 index 00000000000000..7ebc2d05cdd955 --- /dev/null +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/FlagsChangeRebuildTest.cs @@ -0,0 +1,118 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Collections.Generic; +using System.Linq; +using Xunit; +using Xunit.Abstractions; + +#nullable enable + +namespace Wasm.Build.Tests +{ + public class FlagsChangeRebuildTest : NativeRebuildTestsBase + { + public FlagsChangeRebuildTest(ITestOutputHelper output, SharedBuildPerTestClassFixture buildContext) + : base(output, buildContext) + { + } + + public static IEnumerable FlagsChangesForNativeRelinkingData(bool aot) + => ConfigWithAOTData(aot, config: "Release").Multiply( + new object[] { /*cflags*/ "/p:EmccExtraCFlags=-g", /*ldflags*/ "" }, + new object[] { /*cflags*/ "", /*ldflags*/ "/p:EmccExtraLDFlags=-g" }, + new object[] { /*cflags*/ "/p:EmccExtraCFlags=-g", /*ldflags*/ "/p:EmccExtraLDFlags=-g" } + ).WithRunHosts(RunHost.V8).UnwrapItemsAsArrays().Dump(); + + [Theory] + [MemberData(nameof(FlagsChangesForNativeRelinkingData), parameters: /*aot*/ false)] + [MemberData(nameof(FlagsChangesForNativeRelinkingData), parameters: /*aot*/ true)] + public void ExtraEmccFlagsSetButNoRealChange(BuildArgs buildArgs, string extraCFlags, string extraLDFlags, RunHost host, string id) + { + buildArgs = buildArgs with { ProjectName = $"rebuild_flags_{buildArgs.Config}" }; + (buildArgs, BuildPaths paths) = FirstNativeBuild(s_mainReturns42, nativeRelink: true, invariant: false, buildArgs, id); + var pathsDict = GetFilesTable(buildArgs, paths, unchanged: true); + if (extraLDFlags.Length > 0) + pathsDict.UpdateTo(unchanged: false, "dotnet.wasm", "dotnet.js"); + + var originalStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + // Rebuild + + string mainAssembly = $"{buildArgs.ProjectName}.dll"; + string extraBuildArgs = $" {extraCFlags} {extraLDFlags}"; + string output = Rebuild(nativeRelink: true, invariant: false, buildArgs, id, extraBuildArgs: extraBuildArgs, verbosity: "normal"); + + var newStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + CompareStat(originalStat, newStat, pathsDict.Values); + + // cflags: pinvoke get's compiled, but doesn't overwrite pinvoke.o + // and thus doesn't cause relinking + AssertSubstring("pinvoke.c -> pinvoke.o", output, contains: extraCFlags.Length > 0); + + // ldflags: link step args change, so it should trigger relink + AssertSubstring("wasm-opt", output, contains: extraLDFlags.Length > 0); + + if (buildArgs.AOT) + { + // ExtraEmccLDFlags does not affect .bc files + Assert.DoesNotContain("Compiling assembly bitcode files", output); + } + + string runOutput = RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); + AssertSubstring($"Found statically linked AOT module '{Path.GetFileNameWithoutExtension(mainAssembly)}'", runOutput, + contains: buildArgs.AOT); + } + + public static IEnumerable FlagsOnlyChangeData(bool aot) + => ConfigWithAOTData(aot, config: "Release").Multiply( + new object[] { /*cflags*/ "/p:EmccCompileOptimizationFlag=-O1", /*ldflags*/ "" }, + new object[] { /*cflags*/ "", /*ldflags*/ "/p:EmccLinkOptimizationFlag=-O0" } + ).WithRunHosts(RunHost.V8).UnwrapItemsAsArrays().Dump(); + + [Theory] + [MemberData(nameof(FlagsOnlyChangeData), parameters: /*aot*/ false)] + [MemberData(nameof(FlagsOnlyChangeData), parameters: /*aot*/ true)] + public void OptimizationFlagChange(BuildArgs buildArgs, string cflags, string ldflags, RunHost host, string id) + { + // force _WasmDevel=false, so we don't get -O0 + buildArgs = buildArgs with { ProjectName = $"rebuild_flags_{buildArgs.Config}", ExtraBuildArgs = "/p:_WasmDevel=false" }; + (buildArgs, BuildPaths paths) = FirstNativeBuild(s_mainReturns42, nativeRelink: true, invariant: false, buildArgs, id); + + string mainAssembly = $"{buildArgs.ProjectName}.dll"; + var pathsDict = GetFilesTable(buildArgs, paths, unchanged: false); + pathsDict.UpdateTo(unchanged: true, mainAssembly, "icall-table.h", "pinvoke-table.h", "driver-gen.c"); + if (cflags.Length == 0) + pathsDict.UpdateTo(unchanged: true, "pinvoke.o", "corebindings.o", "driver.o"); + + pathsDict.Remove(mainAssembly); + if (buildArgs.AOT) + { + // link optimization flag change affects .bc->.o files too, but + // it might result in only *some* files being *changed, + // so, don't check for those + // Link optimization flag is set to Compile optimization flag, if unset + // so, it affects .bc files too! + foreach (string key in pathsDict.Keys.ToArray()) + { + if (key.EndsWith(".dll.bc", StringComparison.Ordinal) || key.EndsWith(".dll.o", StringComparison.Ordinal)) + pathsDict.Remove(key); + } + } + + var originalStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + // Rebuild + + string output = Rebuild(nativeRelink: true, invariant: false, buildArgs, id, extraBuildArgs: $" {cflags} {ldflags}", verbosity: "normal"); + var newStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + CompareStat(originalStat, newStat, pathsDict.Values); + + string runOutput = RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); + AssertSubstring($"Found statically linked AOT module '{Path.GetFileNameWithoutExtension(mainAssembly)}'", runOutput, + contains: buildArgs.AOT); + } + } +} diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NativeRebuildTestsBase.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NativeRebuildTestsBase.cs new file mode 100644 index 00000000000000..2256fa58386abc --- /dev/null +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NativeRebuildTestsBase.cs @@ -0,0 +1,209 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Collections.Generic; +using System.Linq; +using Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; +using System.Text; + +#nullable enable + +namespace Wasm.Build.Tests +{ + // TODO: test for runtime components + public class NativeRebuildTestsBase : BuildTestBase + { + public NativeRebuildTestsBase(ITestOutputHelper output, SharedBuildPerTestClassFixture buildContext) + : base(output, buildContext) + { + _enablePerTestCleanup = true; + } + + public static IEnumerable NativeBuildData() + { + List data = new(); + // relinking + data.AddRange(GetData(aot: false, nativeRelinking: true, invariant: false)); + data.AddRange(GetData(aot: false, nativeRelinking: true, invariant: true)); + + // aot + data.AddRange(GetData(aot: true, nativeRelinking: false, invariant: false)); + data.AddRange(GetData(aot: true, nativeRelinking: false, invariant: true)); + + return data; + + IEnumerable GetData(bool aot, bool nativeRelinking, bool invariant) + => ConfigWithAOTData(aot) + .Multiply(new object[] { nativeRelinking, invariant }) + .WithRunHosts(RunHost.V8) + .UnwrapItemsAsArrays().ToList().Dump(); + } + + internal (BuildArgs BuildArgs, BuildPaths paths) FirstNativeBuild(string programText, bool nativeRelink, bool invariant, BuildArgs buildArgs, string id, string extraProperties="") + { + buildArgs = GenerateProjectContents(buildArgs, nativeRelink, invariant, extraProperties); + BuildProject(buildArgs, + initProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText), + dotnetWasmFromRuntimePack: false, + hasIcudt: !invariant, + id: id, + createProject: true); + + RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: RunHost.V8, id: id); + return (buildArgs, GetBuildPaths(buildArgs)); + } + + protected string Rebuild(bool nativeRelink, bool invariant, BuildArgs buildArgs, string id, string extraProperties="", string extraBuildArgs="", string? verbosity=null) + { + if (!_buildContext.TryGetBuildFor(buildArgs, out BuildProduct? product)) + throw new XunitException($"Test bug: could not get the build product in the cache"); + + File.Move(product!.LogFile, Path.ChangeExtension(product.LogFile!, ".first.binlog")); + + buildArgs = buildArgs with { ExtraBuildArgs = $"{buildArgs.ExtraBuildArgs} {extraBuildArgs}" }; + var newBuildArgs = GenerateProjectContents(buildArgs, nativeRelink, invariant, extraProperties); + + // key(buildArgs) being changed + _buildContext.RemoveFromCache(product.ProjectDir); + _buildContext.CacheBuild(newBuildArgs, product); + + if (buildArgs.ProjectFileContents != newBuildArgs.ProjectFileContents) + File.WriteAllText(Path.Combine(_projectDir!, $"{buildArgs.ProjectName}.csproj"), buildArgs.ProjectFileContents); + buildArgs = newBuildArgs; + + _testOutput.WriteLine($"{Environment.NewLine}Rebuilding with no changes ..{Environment.NewLine}"); + (_, string output) = BuildProject(buildArgs, + id: id, + dotnetWasmFromRuntimePack: false, + hasIcudt: !invariant, + createProject: false, + useCache: false, + verbosity: verbosity); + + return output; + } + + protected BuildArgs GenerateProjectContents(BuildArgs buildArgs, bool nativeRelink, bool invariant, string extraProperties) + { + StringBuilder propertiesBuilder = new(); + propertiesBuilder.Append("<_WasmDevel>true"); + if (nativeRelink) + propertiesBuilder.Append($"true"); + if (invariant) + propertiesBuilder.Append($"true"); + propertiesBuilder.Append(extraProperties); + + return ExpandBuildArgs(buildArgs, propertiesBuilder.ToString()); + } + + internal void CompareStat(IDictionary oldStat, IDictionary newStat, IEnumerable<(string fullpath, bool unchanged)> expected) + { + StringBuilder msg = new(); + foreach (var expect in expected) + { + string expectFilename = Path.GetFileName(expect.fullpath); + if (!oldStat.TryGetValue(expectFilename, out FileStat? oldFs)) + { + msg.AppendLine($"Could not find an entry for {expectFilename} in old files"); + continue; + } + + if (!newStat.TryGetValue(expectFilename, out FileStat? newFs)) + { + msg.AppendLine($"Could not find an entry for {expectFilename} in new files"); + continue; + } + + bool actualUnchanged = oldFs == newFs; + if (expect.unchanged && !actualUnchanged) + { + msg.AppendLine($"[Expected unchanged file: {expectFilename}]{Environment.NewLine}" + + $" old: {oldFs}{Environment.NewLine}" + + $" new: {newFs}"); + } + else if (!expect.unchanged && actualUnchanged) + { + msg.AppendLine($"[Expected changed file: {expectFilename}]{Environment.NewLine}" + + $" {newFs}"); + } + } + + if (msg.Length > 0) + throw new XunitException($"CompareStat failed:{Environment.NewLine}{msg}"); + } + + internal IDictionary StatFiles(IEnumerable fullpaths) + { + Dictionary table = new(); + foreach (string file in fullpaths) + { + if (File.Exists(file)) + table.Add(Path.GetFileName(file), new FileStat(FullPath: file, Exists: true, LastWriteTimeUtc: File.GetLastWriteTimeUtc(file), Length: new FileInfo(file).Length)); + else + table.Add(Path.GetFileName(file), new FileStat(FullPath: file, Exists: false, LastWriteTimeUtc: DateTime.MinValue, Length: 0)); + } + + return table; + } + + internal BuildPaths GetBuildPaths(BuildArgs buildArgs) + { + string objDir = GetObjDir(buildArgs.Config); + string bundleDir = Path.Combine(GetBinDir(baseDir: _projectDir, config: buildArgs.Config), "AppBundle"); + string wasmDir = Path.Combine(objDir, "wasm"); + + return new BuildPaths(wasmDir, objDir, GetBinDir(buildArgs.Config), bundleDir); + } + + internal IDictionary GetFilesTable(BuildArgs buildArgs, BuildPaths paths, bool unchanged) + { + List files = new() + { + Path.Combine(paths.BinDir, "publish", $"{buildArgs.ProjectName}.dll"), + Path.Combine(paths.ObjWasmDir, "driver.o"), + Path.Combine(paths.ObjWasmDir, "corebindings.o"), + Path.Combine(paths.ObjWasmDir, "pinvoke.o"), + + Path.Combine(paths.ObjWasmDir, "icall-table.h"), + Path.Combine(paths.ObjWasmDir, "pinvoke-table.h"), + Path.Combine(paths.ObjWasmDir, "driver-gen.c"), + + Path.Combine(paths.BundleDir, "dotnet.wasm"), + Path.Combine(paths.BundleDir, "dotnet.js") + }; + + if (buildArgs.AOT) + { + files.AddRange(new[] + { + Path.Combine(paths.ObjWasmDir, $"{buildArgs.ProjectName}.dll.bc"), + Path.Combine(paths.ObjWasmDir, $"{buildArgs.ProjectName}.dll.o"), + + Path.Combine(paths.ObjWasmDir, "System.Private.CoreLib.dll.bc"), + Path.Combine(paths.ObjWasmDir, "System.Private.CoreLib.dll.o"), + }); + } + + var dict = new Dictionary(); + foreach (var file in files) + dict[Path.GetFileName(file)] = (file, unchanged); + + return dict; + } + + protected void AssertSubstring(string substring, string full, bool contains) + { + if (contains) + Assert.Contains(substring, full); + else + Assert.DoesNotContain(substring, full); + } + } + + internal record FileStat (bool Exists, DateTime LastWriteTimeUtc, long Length, string FullPath); + internal record BuildPaths(string ObjWasmDir, string ObjDir, string BinDir, string BundleDir); +} diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NoopNativeRebuildTest.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NoopNativeRebuildTest.cs new file mode 100644 index 00000000000000..1ddbd4467b6206 --- /dev/null +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NoopNativeRebuildTest.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Linq; +using Xunit; +using Xunit.Abstractions; + +#nullable enable + +namespace Wasm.Build.Tests +{ + public class NoopNativeRebuildTest : NativeRebuildTestsBase + { + public NoopNativeRebuildTest(ITestOutputHelper output, SharedBuildPerTestClassFixture buildContext) + : base(output, buildContext) + { + } + + [Theory] + [MemberData(nameof(NativeBuildData))] + public void NoOpRebuildForNativeBuilds(BuildArgs buildArgs, bool nativeRelink, bool invariant, RunHost host, string id) + { + buildArgs = buildArgs with { ProjectName = $"rebuild_noop_{buildArgs.Config}" }; + (buildArgs, BuildPaths paths) = FirstNativeBuild(s_mainReturns42, nativeRelink: nativeRelink, invariant: invariant, buildArgs, id); + + var pathsDict = GetFilesTable(buildArgs, paths, unchanged: true); + var originalStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + Rebuild(nativeRelink, invariant, buildArgs, id); + var newStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + CompareStat(originalStat, newStat, pathsDict.Values); + RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); + } + } +} diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/ReferenceNewAssemblyRebuildTest.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/ReferenceNewAssemblyRebuildTest.cs new file mode 100644 index 00000000000000..d908e287cf8bd1 --- /dev/null +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/ReferenceNewAssemblyRebuildTest.cs @@ -0,0 +1,57 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.IO; +using System.Linq; +using Xunit; +using Xunit.Abstractions; + +#nullable enable + +namespace Wasm.Build.Tests +{ + public class ReferenceNewAssemblyRebuildTest : NativeRebuildTestsBase + { + public ReferenceNewAssemblyRebuildTest(ITestOutputHelper output, SharedBuildPerTestClassFixture buildContext) + : base(output, buildContext) + { + } + + [Theory] + [MemberData(nameof(NativeBuildData))] + public void ReferenceNewAssembly(BuildArgs buildArgs, bool nativeRelink, bool invariant, RunHost host, string id) + { + buildArgs = buildArgs with { ProjectName = $"rebuild_tasks_{buildArgs.Config}" }; + (buildArgs, BuildPaths paths) = FirstNativeBuild(s_mainReturns42, nativeRelink, invariant: invariant, buildArgs, id); + + var pathsDict = GetFilesTable(buildArgs, paths, unchanged: false); + pathsDict.UpdateTo(unchanged: true, "corebindings.o"); + if (!buildArgs.AOT) // relinking + pathsDict.UpdateTo(unchanged: true, "driver-gen.c"); + + var originalStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + string programText = + @$" + using System; + using System.Text.Json; + public class Test + {{ + public static int Main() + {{" + + @" string json = ""{ \""name\"": \""value\"" }"";" + + @" var jdoc = JsonDocument.Parse($""{json}"", new JsonDocumentOptions());" + + @$" Console.WriteLine($""json: {{jdoc}}""); + return 42; + }} + }}"; + File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText); + + Rebuild(nativeRelink, invariant, buildArgs, id); + var newStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + CompareStat(originalStat, newStat, pathsDict.Values); + RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); + } + } +} diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/SimpleSourceChangeRebuildTest.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/SimpleSourceChangeRebuildTest.cs new file mode 100644 index 00000000000000..7f51447cb9e62a --- /dev/null +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/SimpleSourceChangeRebuildTest.cs @@ -0,0 +1,55 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.IO; +using System.Linq; +using Xunit; +using Xunit.Abstractions; + +#nullable enable + +namespace Wasm.Build.Tests +{ + public class SimpleSourceChangeRebuildTest : NativeRebuildTestsBase + { + public SimpleSourceChangeRebuildTest(ITestOutputHelper output, SharedBuildPerTestClassFixture buildContext) + : base(output, buildContext) + { + } + + [Theory] + [MemberData(nameof(NativeBuildData))] + public void SimpleStringChangeInSource(BuildArgs buildArgs, bool nativeRelink, bool invariant, RunHost host, string id) + { + buildArgs = buildArgs with { ProjectName = $"rebuild_simple_{buildArgs.Config}" }; + (buildArgs, BuildPaths paths) = FirstNativeBuild(s_mainReturns42, nativeRelink, invariant: invariant, buildArgs, id); + + string mainAssembly = $"{buildArgs.ProjectName}.dll"; + var pathsDict = GetFilesTable(buildArgs, paths, unchanged: true); + pathsDict.UpdateTo(unchanged: false, mainAssembly); + pathsDict.UpdateTo(unchanged: !buildArgs.AOT, "dotnet.wasm", "dotnet.js"); + + if (buildArgs.AOT) + pathsDict.UpdateTo(unchanged: false, $"{mainAssembly}.bc", $"{mainAssembly}.o"); + + var originalStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + // Changes + string mainResults55 = @" + public class TestClass { + public static int Main() + { + return 55; + } + }"; + File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), mainResults55); + + // Rebuild + Rebuild(nativeRelink, invariant, buildArgs, id); + var newStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + CompareStat(originalStat, newStat, pathsDict.Values); + RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 55, host: host, id: id); + } + } +} From 2147a148452702d71287d106112a8e042b2b71c9 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 11 Aug 2021 11:37:02 -0400 Subject: [PATCH 15/23] remove non-test classes --- eng/testing/scenarios/BuildWasmAppsJobsList.txt | 7 ------- 1 file changed, 7 deletions(-) diff --git a/eng/testing/scenarios/BuildWasmAppsJobsList.txt b/eng/testing/scenarios/BuildWasmAppsJobsList.txt index ae5981223de3de..ba322274dfe62c 100644 --- a/eng/testing/scenarios/BuildWasmAppsJobsList.txt +++ b/eng/testing/scenarios/BuildWasmAppsJobsList.txt @@ -1,21 +1,14 @@ BlazorWasmTests -BuildAndRunAttribute -BuildEnvironment -BuildTestBase -DotNetCommand FlagsChangeRebuildTest InvariantGlobalizationTests LocalEMSDKTests MainWithArgsTests NativeBuildTests NativeLibraryTests -NativeRebuildTestsBase NoopNativeRebuildTest RebuildTests ReferenceNewAssemblyRebuildTest SatelliteAssembliesTests -SharedBuildPerTestClassFixture SimpleSourceChangeRebuildTest -ToolCommand WasmBuildAppTest WorkloadTests From ef97caf54647af22a9e37a3ac88a7846439e0899 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 11 Aug 2021 12:21:54 -0400 Subject: [PATCH 16/23] add back builds This reverts commit b008130a7886c2e2b9f16c83641c1b8c936082f6. --- eng/pipelines/common/platform-matrix.yml | 635 +++++++++++++++++++++++ 1 file changed, 635 insertions(+) diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index 6dab0ab529fe2a..15fe948e92967e 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -26,6 +26,185 @@ parameters: jobs: +# Linux arm +- ${{ if or(containsValue(parameters.platforms, 'Linux_arm'), in(parameters.platformGroup, 'all', 'gcstress')) }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Linux + archType: arm + targetRid: linux-arm + platform: Linux_arm + container: + image: ubuntu-16.04-cross-20210719121212-8a8d3be + registry: mcr + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + crossBuild: true + crossrootfsDir: '/crossrootfs/arm' + ${{ insert }}: ${{ parameters.jobParameters }} + +# Linux arm64 + +- ${{ if or(containsValue(parameters.platforms, 'Linux_arm64'), in(parameters.platformGroup, 'all', 'gcstress')) }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Linux + archType: arm64 + targetRid: linux-arm64 + platform: Linux_arm64 + container: + image: ubuntu-16.04-cross-arm64-20210719121212-8a8d3be + registry: mcr + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + crossBuild: true + crossrootfsDir: '/crossrootfs/arm64' + ${{ insert }}: ${{ parameters.jobParameters }} + +# Linux musl x64 + +- ${{ if or(containsValue(parameters.platforms, 'Linux_musl_x64'), eq(parameters.platformGroup, 'all')) }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Linux + osSubgroup: _musl + archType: x64 + targetRid: linux-musl-x64 + platform: Linux_musl_x64 + container: + image: alpine-3.9-WithNode-20210714125437-9b5bbc2 + registry: mcr + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Linux musl arm + +- ${{ if or(containsValue(parameters.platforms, 'Linux_musl_arm'), eq(parameters.platformGroup, 'all')) }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Linux + osSubgroup: _musl + archType: arm + targetRid: linux-musl-arm + platform: Linux_musl_arm + container: + image: ubuntu-16.04-cross-arm-alpine-20210719121212-044d5b9 + registry: mcr + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + crossBuild: true + crossrootfsDir: '/crossrootfs/arm' + ${{ insert }}: ${{ parameters.jobParameters }} + +# Linux musl arm64 + +- ${{ if or(containsValue(parameters.platforms, 'Linux_musl_arm64'), eq(parameters.platformGroup, 'all')) }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Linux + osSubgroup: _musl + archType: arm64 + targetRid: linux-musl-arm64 + platform: Linux_musl_arm64 + container: + image: ubuntu-16.04-cross-arm64-alpine-20210719121212-b2c2436 + registry: mcr + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + crossBuild: true + crossrootfsDir: '/crossrootfs/arm64' + ${{ insert }}: ${{ parameters.jobParameters }} + +# Linux x64 + +- ${{ if or(containsValue(parameters.platforms, 'Linux_x64'), containsValue(parameters.platforms, 'CoreClrTestBuildHost'), in(parameters.platformGroup, 'all', 'gcstress')) }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Linux + archType: x64 + targetRid: linux-x64 + platform: Linux_x64 + container: + image: centos-7-20210714125435-9b5bbc2 + registry: mcr + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Linux x64 Source Build + +- ${{ if containsValue(parameters.platforms, 'SourceBuild_Linux_x64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Linux + archType: x64 + targetRid: linux-x64 + platform: Linux_x64 + container: + image: centos-7-source-build-20210714125450-5d87b80 + registry: mcr + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + buildingOnSourceBuildImage: true + # WebAssembly - ${{ if containsValue(parameters.platforms, 'Browser_wasm') }}: @@ -71,3 +250,459 @@ jobs: platforms: ${{ parameters.platforms }} ${{ insert }}: ${{ parameters.jobParameters }} +# FreeBSD +- ${{ if containsValue(parameters.platforms, 'FreeBSD_x64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: FreeBSD + archType: x64 + targetRid: freebsd-x64 + platform: FreeBSD_x64 + container: + image: ubuntu-18.04-cross-freebsd-11-20200407092345-a84b0d2 + registry: mcr + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + buildConfig: ${{ parameters.buildConfig }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + crossBuild: true + crossrootfsDir: '/crossrootfs/x64' + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Android x64 + +- ${{ if containsValue(parameters.platforms, 'Android_x64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Android + archType: x64 + targetRid: android-x64 + platform: Android_x64 + container: + image: ubuntu-18.04-android-20200422191843-e2c3f83 + registry: mcr + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Android x86 + +- ${{ if containsValue(parameters.platforms, 'Android_x86') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Android + archType: x86 + targetRid: android-x86 + platform: Android_x86 + container: + image: ubuntu-18.04-android-20200422191843-e2c3f83 + registry: mcr + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Android arm + +- ${{ if containsValue(parameters.platforms, 'Android_arm') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Android + archType: arm + targetRid: android-arm + platform: Android_arm + container: + image: ubuntu-18.04-android-20200422191843-e2c3f83 + registry: mcr + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Android arm64 + +- ${{ if containsValue(parameters.platforms, 'Android_arm64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Android + archType: arm64 + targetRid: android-arm64 + platform: Android_arm64 + container: + image: ubuntu-18.04-android-20200422191843-e2c3f83 + registry: mcr + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Mac Catalyst x64 + +- ${{ if containsValue(parameters.platforms, 'MacCatalyst_x64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: MacCatalyst + archType: x64 + targetRid: maccatalyst-x64 + platform: MacCatalyst_x64 + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Mac Catalyst arm64 + +- ${{ if containsValue(parameters.platforms, 'MacCatalyst_arm64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: MacCatalyst + archType: arm64 + targetRid: maccatalyst-arm64 + platform: MacCatalyst_arm64 + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# tvOS arm64 + +- ${{ if containsValue(parameters.platforms, 'tvOS_arm64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: tvOS + archType: arm64 + targetRid: tvos-arm64 + platform: tvOS_arm64 + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# tvOS Simulator x64 + +- ${{ if containsValue(parameters.platforms, 'tvOSSimulator_x64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: tvOSSimulator + archType: x64 + targetRid: tvossimulator-x64 + platform: tvOSSimulator_x64 + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# tvOS Simulator arm64 + +- ${{ if containsValue(parameters.platforms, 'tvOSSimulator_arm64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: tvOSSimulator + archType: arm64 + targetRid: tvossimulator-arm64 + platform: tvOSSimulator_arm64 + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# iOS arm + +- ${{ if containsValue(parameters.platforms, 'iOS_arm') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: iOS + archType: arm + targetRid: ios-arm + platform: iOS_arm + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# iOS arm64 + +- ${{ if containsValue(parameters.platforms, 'iOS_arm64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: iOS + archType: arm64 + targetRid: ios-arm64 + platform: iOS_arm64 + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# iOS Simulator x64 + +- ${{ if containsValue(parameters.platforms, 'iOSSimulator_x64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: iOSSimulator + archType: x64 + targetRid: iossimulator-x64 + platform: iOSSimulator_x64 + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# iOS Simulator x86 + +- ${{ if containsValue(parameters.platforms, 'iOSSimulator_x86') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: iOSSimulator + archType: x86 + targetRid: iossimulator-x86 + platform: iOSsimulator_x86 + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + managedTestBuildOsGroup: OSX + ${{ insert }}: ${{ parameters.jobParameters }} + +# iOS Simulator arm64 + +- ${{ if containsValue(parameters.platforms, 'iOSSimulator_arm64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: iOSSimulator + archType: arm64 + targetRid: iossimulator-arm64 + platform: iOSSimulator_arm64 + jobParameters: + runtimeFlavor: mono + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# macOS arm64 + +- ${{ if containsValue(parameters.platforms, 'OSX_arm64') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: OSX + archType: arm64 + targetRid: osx-arm64 + platform: OSX_arm64 + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + crossBuild: true + ${{ insert }}: ${{ parameters.jobParameters }} + +# macOS x64 + +- ${{ if or(containsValue(parameters.platforms, 'OSX_x64'), eq(parameters.platformGroup, 'all')) }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: OSX + archType: x64 + targetRid: osx-x64 + platform: OSX_x64 + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Windows x64 + +- ${{ if or(containsValue(parameters.platforms, 'windows_x64'), in(parameters.platformGroup, 'all', 'gcstress')) }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: windows + archType: x64 + targetRid: win-x64 + platform: windows_x64 + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Windows x86 + +- ${{ if or(containsValue(parameters.platforms, 'windows_x86'), in(parameters.platformGroup, 'all', 'gcstress')) }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: windows + archType: x86 + targetRid: win-x86 + platform: windows_x86 + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Windows arm +- ${{ if or(containsValue(parameters.platforms, 'windows_arm'), eq(parameters.platformGroup, 'all')) }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: windows + archType: arm + targetRid: win-arm + platform: windows_arm + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} + +# Windows arm64 + +- ${{ if or(containsValue(parameters.platforms, 'windows_arm64'), in(parameters.platformGroup, 'all', 'gcstress')) }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: windows + archType: arm64 + targetRid: win-arm64 + platform: windows_arm64 + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + ${{ insert }}: ${{ parameters.jobParameters }} From fd62b0078ba8456d15c0c4d12ac6d3a9d35ae164 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 11 Aug 2021 17:48:47 -0400 Subject: [PATCH 17/23] MonoAOTCompiler: make cache optional --- src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 53 ++++++++++++-------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index 098b8a3dbe20fa..bf18635118b34c 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -185,8 +185,10 @@ public class MonoAOTCompiler : Microsoft.Build.Utilities.Task /// public string? LLVMDebug { get; set; } = "nodebug"; - [NotNull] - [Required] + /// + /// File used to track hashes of assemblies, to act as a cache + /// Output files don't get written, if they haven't changed + /// public string? CacheFilePath { get; set; } [Output] @@ -323,24 +325,7 @@ public override bool Execute() if (AdditionalAssemblySearchPaths != null) monoPaths = string.Join(Path.PathSeparator.ToString(), AdditionalAssemblySearchPaths); - if (File.Exists(CacheFilePath)) - { - _oldCache = (CompilerCache?)JsonSerializer.Deserialize(File.ReadAllText(CacheFilePath!), - typeof(CompilerCache), - new JsonSerializerOptions()); - _oldCache ??= new(); - } - else - { - _oldCache = new(); - } - - _newCache = new(); - foreach (var assemblyItem in Assemblies) - { - string newHash = Utils.ComputeHash(assemblyItem.GetMetadata("FullPath")); - _newCache.AssemblyHashes[assemblyItem.ItemSpec] = newHash; - } + InitCache(); //FIXME: check the nothing changed at all case @@ -875,6 +860,34 @@ private bool GenerateAotModulesTable(ITaskItem[] assemblies, string[]? profilers return true; } + private void InitCache() + { + if (string.IsNullOrEmpty(CacheFilePath)) + { + Log.LogMessage(MessageImportance.Low, $"Disabling cache, because {nameof(CacheFilePath)} is not set"); + return; + } + + if (File.Exists(CacheFilePath)) + { + _oldCache = (CompilerCache?)JsonSerializer.Deserialize(File.ReadAllText(CacheFilePath!), + typeof(CompilerCache), + new JsonSerializerOptions()); + _oldCache ??= new(); + } + else + { + _oldCache = new(); + } + + _newCache = new(); + foreach (var assemblyItem in Assemblies) + { + string newHash = Utils.ComputeHash(assemblyItem.GetMetadata("FullPath")); + _newCache.AssemblyHashes[assemblyItem.ItemSpec] = newHash; + } + } + private bool TryGetAssemblyName(string asmPath, [NotNullWhen(true)] out string? assemblyName) { assemblyName = null; From 4ee7572cf62183d432f27da59814aa2149d5b484 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 11 Aug 2021 18:44:43 -0400 Subject: [PATCH 18/23] MonoAOTCompiler: handle the case where we have a cache entry, but the file on disk doesn't exist --- src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 133 +++++++++++-------- src/tasks/WasmAppBuilder/EmccCompile.cs | 4 +- 2 files changed, 80 insertions(+), 57 deletions(-) diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index bf18635118b34c..3cd4fff67dc490 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -653,36 +653,9 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) return false; } - if (Path.GetExtension(tmpOutputFileName) == ".tmp") - { - string finalPath = Path.Combine(Path.GetDirectoryName(tmpOutputFileName) ?? string.Empty, - Path.GetFileNameWithoutExtension(tmpOutputFileName)); - - if (HaveAsmDependenciesChanged(assembly, depfile, out string? cause)) - { - if (File.Exists(finalPath)) - File.Delete(finalPath); - - File.Copy(tmpOutputFileName, finalPath); - - Log.LogMessage(MessageImportance.Low, $"Copying {tmpOutputFileName} to {finalPath} because {cause}"); - _fileWrites.Add(finalPath); - - int count = Interlocked.Increment(ref _numCompiled); - Log.LogMessage(MessageImportance.High, $"[{count}/{_totalNumAssemblies}] {assemblyFilename} -> {Path.GetFileName(finalPath)}"); - } - else - { - Log.LogMessage(MessageImportance.Low, $"Skipping copying over {finalPath} as the contents are unchanged"); - } - } - else - { - Log.LogWarning($"Bug: expected tmpOutputFileName to end in .tmp: {tmpOutputFileName}"); - } - - File.Delete(responseFilePath); + CopyOutputFileIfChanged(assembly, tmpOutputFileName, depfile); File.Delete(tmpOutputFileName); + File.Delete(responseFilePath); File.Delete(depfile); compiledAssemblies.GetOrAdd(aotAssembly.ItemSpec, aotAssembly); @@ -725,47 +698,97 @@ private void PrecompileLibraryParallel(ITaskItem assemblyItem, string? monoPaths } } - private bool HaveAsmDependenciesChanged(string assemblyFileName, string depfile, [NotNullWhen(true)] out string? cause) + private void CopyOutputFileIfChanged(string assemblyPath, string tmpOutputFileName, string depfile) { - cause = null; - if (!File.Exists(depfile)) + if (Path.GetExtension(tmpOutputFileName) != ".tmp") { - cause = $"couldn't find depfile {depfile}"; - return true; + Log.LogWarning($"Bug: expected tmpOutputFileName to end in .tmp: {tmpOutputFileName}"); + return; } - string deps = File.ReadAllText(depfile); - string[] parts = deps.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); - if (parts.Length < 2) + string finalPath = Path.Combine(Path.GetDirectoryName(tmpOutputFileName) ?? string.Empty, + Path.GetFileNameWithoutExtension(tmpOutputFileName)); + + if (!ShouldCopy(out string? cause)) { - cause = $"{depfile} had invalid format"; - return true; + Log.LogMessage(MessageImportance.Low, $"Skipping copying over {finalPath} as the contents are unchanged"); + return; } - Log.LogMessage(MessageImportance.Low, $"{assemblyFileName} depends on {deps}"); - string? changedAsm = parts.Skip(1).FirstOrDefault(asm => HasHashChanged(asm.Trim())); - if (changedAsm != null) - cause = changedAsm == assemblyFileName ? "it changed" : $"dependency {changedAsm} changed."; + if (File.Exists(finalPath)) + File.Delete(finalPath); - return changedAsm != null; - } + File.Copy(tmpOutputFileName, finalPath); - private bool HasHashChanged(string assemblyFileName) - { - if (!_oldCache!.AssemblyHashes.TryGetValue(assemblyFileName, out string? oldHash)) + Log.LogMessage(MessageImportance.Low, $"Copying {tmpOutputFileName} to {finalPath} because {cause}"); + _fileWrites.Add(finalPath); + + int count = Interlocked.Increment(ref _numCompiled); + Log.LogMessage(MessageImportance.High, $"[{count}/{_totalNumAssemblies}] {assemblyPath} -> {Path.GetFileName(finalPath)}"); + + bool ShouldCopy([NotNullWhen(true)] out string? cause) { - // we haven't seen this file before + cause = null; + + if (_oldCache == null || _newCache == null) + { + cause = "cache is disabled"; + return true; + } + + if (!File.Exists(assemblyPath)) + { + cause = $"the output file didn't exist"; + return true; + } + + if (!File.Exists(depfile)) + { + cause = $"couldn't find depfile {depfile}"; + return true; + } + + string deps = File.ReadAllText(depfile); + string[] parts = deps.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); + if (parts.Length < 2) + { + cause = $"{depfile} had invalid format"; + return true; + } + + Log.LogMessage(MessageImportance.Low, $"{assemblyPath} depends on {deps}"); + string? changedAsm = parts.Skip(1).FirstOrDefault(asm => HasHashChanged(asm.Trim())); + if (changedAsm == null) + return false; + + cause = changedAsm == assemblyPath + ? "it changed" + : $"dependency {changedAsm} changed."; return true; } - if (!_newCache!.AssemblyHashes.TryGetValue(assemblyFileName, out string? newHash)) + bool HasHashChanged(string assemblyPath) { - // um should not happen - throw new Exception($"BUG? can't find new hash for {assemblyFileName}"); - } + if (_oldCache == null || _newCache == null) + { + // cache disabled + return true; + } - var res = oldHash != newHash; - return res; + if (!_oldCache!.AssemblyHashes.TryGetValue(assemblyPath, out string? oldHash)) + { + // we haven't seen this file before + return true; + } + + if (!_newCache!.AssemblyHashes.TryGetValue(assemblyPath, out string? newHash)) + { + // um should not happen + throw new Exception($"BUG? can't find new hash for {assemblyPath}"); + } + + return oldHash != newHash; + } } private bool GenerateAotModulesTable(ITaskItem[] assemblies, string[]? profilers, string outputFile) diff --git a/src/tasks/WasmAppBuilder/EmccCompile.cs b/src/tasks/WasmAppBuilder/EmccCompile.cs index 55faa121d9db4b..e869e98e8564ff 100644 --- a/src/tasks/WasmAppBuilder/EmccCompile.cs +++ b/src/tasks/WasmAppBuilder/EmccCompile.cs @@ -240,8 +240,8 @@ bool IsNewerThanOutput(string inFile, string outFile, out string reason) return true; } - DateTime lastWriteTimeSrc= File.GetLastWriteTimeUtc(inFile); - DateTime lastWriteTimeDst= File.GetLastWriteTimeUtc(outFile); + DateTime lastWriteTimeSrc = File.GetLastWriteTimeUtc(inFile); + DateTime lastWriteTimeDst = File.GetLastWriteTimeUtc(outFile); if (lastWriteTimeSrc > lastWriteTimeDst) { From f4f587cd55b7d47705e0fb2c6dffdd3d57165eff Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 11 Aug 2021 19:38:14 -0400 Subject: [PATCH 19/23] Fix aot compiler task output --- src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index 3cd4fff67dc490..b838311ca9fec4 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -724,7 +724,7 @@ private void CopyOutputFileIfChanged(string assemblyPath, string tmpOutputFileNa _fileWrites.Add(finalPath); int count = Interlocked.Increment(ref _numCompiled); - Log.LogMessage(MessageImportance.High, $"[{count}/{_totalNumAssemblies}] {assemblyPath} -> {Path.GetFileName(finalPath)}"); + Log.LogMessage(MessageImportance.High, $"[{count}/{_totalNumAssemblies}] {Path.GetFileName(assemblyPath)} -> {Path.GetFileName(finalPath)}"); bool ShouldCopy([NotNullWhen(true)] out string? cause) { From 6ecf7b22f64ed9b75e23b8bab694735f3be2a655 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Thu, 12 Aug 2021 01:22:01 -0400 Subject: [PATCH 20/23] MonoAOTCompiler: Use hashes of .bc files instead of assemblies `--depfile` isn't supported on aot config used by android, and fails with: ``` * Assertion at /__w/1/s/src/mono/mono/mini/aot-compiler.c:14216, condition `acfg->aot_opts.llvm_only && acfg->aot_opts.asm_only && acfg->aot_opts.llvm_outfile' not met ``` Instead, use hashes of the .bc.tmp files generated, with the existing .bc files. --- src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 144 +++++++------------ 1 file changed, 53 insertions(+), 91 deletions(-) diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index b838311ca9fec4..5cacd2e2fabf02 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -202,8 +202,10 @@ public class MonoAOTCompiler : Microsoft.Build.Utilities.Task private MonoAotOutputType parsedOutputType; private MonoAotLibraryFormat parsedLibraryFormat; private MonoAotModulesTableLanguage parsedAotModulesTableLanguage; + private CompilerCache? _newCache; private CompilerCache? _oldCache; + private bool _useCache => _newCache != null && _oldCache != null; private int _numCompiled; private int _totalNumAssemblies; @@ -362,9 +364,12 @@ public override bool Execute() if (numUnchanged > 0 && numUnchanged != _totalNumAssemblies) Log.LogMessage(MessageImportance.High, $"[{numUnchanged}/{_totalNumAssemblies}] skipped unchanged assemblies."); - var json = JsonSerializer.Serialize (_newCache, new JsonSerializerOptions { WriteIndented = true }); - File.WriteAllText(CacheFilePath!, json); - _fileWrites.Add(CacheFilePath!); + if (_useCache) + { + var json = JsonSerializer.Serialize (_newCache, new JsonSerializerOptions { WriteIndented = true }); + File.WriteAllText(CacheFilePath!, json); + _fileWrites.Add(CacheFilePath!); + } CompiledAssemblies = ConvertAssembliesDictToOrderedList(compiledAssemblies, Assemblies).ToArray(); FileWrites = _fileWrites.ToArray(); @@ -442,7 +447,7 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) aotArgs.Add("dedup-skip"); } - string tmpOutputFileName; + string tmpOutputFile; // compute output mode and file names if (parsedAotMode == MonoAotMode.LLVMOnly || parsedAotMode == MonoAotMode.LLVMOnlyInterp) @@ -450,7 +455,7 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) aotArgs.Add("llvmonly"); string llvmBitcodeFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.bc")); - tmpOutputFileName = llvmBitcodeFile + ".tmp"; + tmpOutputFile = llvmBitcodeFile + ".tmp"; aotAssembly.SetMetadata("LlvmBitcodeFile", llvmBitcodeFile); if (parsedAotMode == MonoAotMode.LLVMOnlyInterp) @@ -461,11 +466,11 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) if (parsedOutputType == MonoAotOutputType.AsmOnly) { aotArgs.Add("asmonly"); - aotArgs.Add($"llvm-outfile={tmpOutputFileName}"); + aotArgs.Add($"llvm-outfile={tmpOutputFile}"); } else { - aotArgs.Add($"outfile={tmpOutputFileName}"); + aotArgs.Add($"outfile={tmpOutputFile}"); } } else @@ -490,8 +495,8 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) case MonoAotOutputType.ObjectFile: { string objectFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.o")); - tmpOutputFileName = objectFile + ".tmp"; - aotArgs.Add($"outfile={tmpOutputFileName}"); + tmpOutputFile = objectFile + ".tmp"; + aotArgs.Add($"outfile={tmpOutputFile}"); aotAssembly.SetMetadata("ObjectFile", objectFile); } break; @@ -501,8 +506,8 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) aotArgs.Add("asmonly"); string assemblerFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.s")); - tmpOutputFileName = assemblerFile + ".tmp"; - aotArgs.Add($"outfile={tmpOutputFileName}"); + tmpOutputFile = assemblerFile + ".tmp"; + aotArgs.Add($"outfile={tmpOutputFile}"); aotAssembly.SetMetadata("AssemblerFile", assemblerFile); } break; @@ -517,9 +522,9 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) }; string libraryFileName = $"{LibraryFilePrefix}{assemblyFilename}{extension}"; string libraryFilePath = Path.Combine(OutputDir, libraryFileName); - tmpOutputFileName = libraryFilePath + ".tmp"; + tmpOutputFile = libraryFilePath + ".tmp"; - aotArgs.Add($"outfile={tmpOutputFileName}"); + aotArgs.Add($"outfile={tmpOutputFile}"); aotAssembly.SetMetadata("LibraryFile", libraryFilePath); } break; @@ -531,8 +536,8 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) if (UseLLVM) { string llvmObjectFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll-llvm.o")); - tmpOutputFileName = llvmObjectFile; - aotArgs.Add($"llvm-outfile={tmpOutputFileName}"); + tmpOutputFile = llvmObjectFile; + aotArgs.Add($"llvm-outfile={tmpOutputFile}"); aotAssembly.SetMetadata("LlvmObjectFile", llvmObjectFile); } } @@ -559,9 +564,6 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) } } - string depfile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".depfile")); - aotArgs.Add($"depfile={depfile}"); - // we need to quote the entire --aot arguments here to make sure it is parsed // on Windows as one argument. Otherwise it will be split up into multiple // values, which wont work. @@ -647,16 +649,15 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) return false; } - if (!File.Exists(tmpOutputFileName)) + if (!File.Exists(tmpOutputFile)) { - Log.LogError($"Precompiling failed for {assembly}. Could not find output file {tmpOutputFileName}"); + Log.LogError($"Precompiling failed for {assembly}. Could not find output file {tmpOutputFile}"); return false; } - CopyOutputFileIfChanged(assembly, tmpOutputFileName, depfile); - File.Delete(tmpOutputFileName); + CopyOutputFileIfChanged(assembly, tmpOutputFile); + File.Delete(tmpOutputFile); File.Delete(responseFilePath); - File.Delete(depfile); compiledAssemblies.GetOrAdd(aotAssembly.ItemSpec, aotAssembly); return true; @@ -698,96 +699,65 @@ private void PrecompileLibraryParallel(ITaskItem assemblyItem, string? monoPaths } } - private void CopyOutputFileIfChanged(string assemblyPath, string tmpOutputFileName, string depfile) + private void CopyOutputFileIfChanged(string assemblyPath, string tmpOutputFile) { - if (Path.GetExtension(tmpOutputFileName) != ".tmp") + if (Path.GetExtension(tmpOutputFile) != ".tmp") { - Log.LogWarning($"Bug: expected tmpOutputFileName to end in .tmp: {tmpOutputFileName}"); + Log.LogWarning($"Bug: expected tmpOutputFile to end in .tmp: {tmpOutputFile}"); return; } - string finalPath = Path.Combine(Path.GetDirectoryName(tmpOutputFileName) ?? string.Empty, - Path.GetFileNameWithoutExtension(tmpOutputFileName)); + string finalOutputFile = Path.Combine(Path.GetDirectoryName(tmpOutputFile) ?? string.Empty, + Path.GetFileNameWithoutExtension(tmpOutputFile)); if (!ShouldCopy(out string? cause)) { - Log.LogMessage(MessageImportance.Low, $"Skipping copying over {finalPath} as the contents are unchanged"); + Log.LogMessage(MessageImportance.Low, $"Skipping copying over {finalOutputFile} as the contents are unchanged"); return; } - if (File.Exists(finalPath)) - File.Delete(finalPath); + if (File.Exists(finalOutputFile)) + File.Delete(finalOutputFile); - File.Copy(tmpOutputFileName, finalPath); + File.Copy(tmpOutputFile, finalOutputFile); - Log.LogMessage(MessageImportance.Low, $"Copying {tmpOutputFileName} to {finalPath} because {cause}"); - _fileWrites.Add(finalPath); + Log.LogMessage(MessageImportance.Low, $"Copying {tmpOutputFile} to {finalOutputFile} because {cause}"); + _fileWrites.Add(finalOutputFile); int count = Interlocked.Increment(ref _numCompiled); - Log.LogMessage(MessageImportance.High, $"[{count}/{_totalNumAssemblies}] {Path.GetFileName(assemblyPath)} -> {Path.GetFileName(finalPath)}"); + Log.LogMessage(MessageImportance.High, $"[{count}/{_totalNumAssemblies}] {Path.GetFileName(assemblyPath)} -> {Path.GetFileName(finalOutputFile)}"); bool ShouldCopy([NotNullWhen(true)] out string? cause) { cause = null; - if (_oldCache == null || _newCache == null) + if (!_useCache) { + // cache disabled cause = "cache is disabled"; return true; } - if (!File.Exists(assemblyPath)) - { - cause = $"the output file didn't exist"; - return true; - } - - if (!File.Exists(depfile)) - { - cause = $"couldn't find depfile {depfile}"; - return true; - } + string newHash = Utils.ComputeHash(tmpOutputFile); + _newCache!.FileHashes[finalOutputFile] = newHash; - string deps = File.ReadAllText(depfile); - string[] parts = deps.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); - if (parts.Length < 2) + if (!File.Exists(finalOutputFile)) { - cause = $"{depfile} had invalid format"; + cause = $"the output file didn't exist"; return true; } - Log.LogMessage(MessageImportance.Low, $"{assemblyPath} depends on {deps}"); - string? changedAsm = parts.Skip(1).FirstOrDefault(asm => HasHashChanged(asm.Trim())); - if (changedAsm == null) - return false; - - cause = changedAsm == assemblyPath - ? "it changed" - : $"dependency {changedAsm} changed."; - return true; - } - - bool HasHashChanged(string assemblyPath) - { - if (_oldCache == null || _newCache == null) - { - // cache disabled - return true; - } + string? oldHash; + if (!_oldCache!.FileHashes.TryGetValue(finalOutputFile, out oldHash)) + oldHash = Utils.ComputeHash(finalOutputFile); - if (!_oldCache!.AssemblyHashes.TryGetValue(assemblyPath, out string? oldHash)) + if (oldHash != newHash) { - // we haven't seen this file before + cause = $"hash for the file changed"; return true; } - if (!_newCache!.AssemblyHashes.TryGetValue(assemblyPath, out string? newHash)) - { - // um should not happen - throw new Exception($"BUG? can't find new hash for {assemblyPath}"); - } - - return oldHash != newHash; + return false; } } @@ -883,12 +853,12 @@ private bool GenerateAotModulesTable(ITaskItem[] assemblies, string[]? profilers return true; } - private void InitCache() + private bool InitCache() { if (string.IsNullOrEmpty(CacheFilePath)) { Log.LogMessage(MessageImportance.Low, $"Disabling cache, because {nameof(CacheFilePath)} is not set"); - return; + return false; } if (File.Exists(CacheFilePath)) @@ -896,19 +866,11 @@ private void InitCache() _oldCache = (CompilerCache?)JsonSerializer.Deserialize(File.ReadAllText(CacheFilePath!), typeof(CompilerCache), new JsonSerializerOptions()); - _oldCache ??= new(); - } - else - { - _oldCache = new(); } + _oldCache ??= new(); _newCache = new(); - foreach (var assemblyItem in Assemblies) - { - string newHash = Utils.ComputeHash(assemblyItem.GetMetadata("FullPath")); - _newCache.AssemblyHashes[assemblyItem.ItemSpec] = newHash; - } + return true; } private bool TryGetAssemblyName(string asmPath, [NotNullWhen(true)] out string? assemblyName) @@ -985,5 +947,5 @@ public enum MonoAotModulesTableLanguage internal class CompilerCache { [JsonPropertyName("assembly_hashes")] - public ConcurrentDictionary AssemblyHashes { get; set; } = new(); + public ConcurrentDictionary FileHashes { get; set; } = new(); } From 609d9b37d8ee2619514e2e53f061c421a9dbe485 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Thu, 12 Aug 2021 04:25:53 -0400 Subject: [PATCH 21/23] MonoAOTCompiler: Support more than one output file The earlier implementation assumed that there would be only one output file. But in some cases (eg. android), there are more than one, like `.s`, `.dll-llvm.o`. --- src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 116 +++++++++++-------- 1 file changed, 67 insertions(+), 49 deletions(-) diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index 5cacd2e2fabf02..0820f334048999 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -385,6 +385,7 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) var aotArgs = new List(); var processArgs = new List(); bool isDedup = assembly == DedupAssembly; + List proxyFiles = new(capacity: 5); string msgPrefix = $"[{Path.GetFileName(assembly)}] "; var a = assemblyItem.GetMetadata("AotArguments"); @@ -447,16 +448,15 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) aotArgs.Add("dedup-skip"); } - string tmpOutputFile; - // compute output mode and file names if (parsedAotMode == MonoAotMode.LLVMOnly || parsedAotMode == MonoAotMode.LLVMOnlyInterp) { aotArgs.Add("llvmonly"); string llvmBitcodeFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.bc")); - tmpOutputFile = llvmBitcodeFile + ".tmp"; - aotAssembly.SetMetadata("LlvmBitcodeFile", llvmBitcodeFile); + ProxyFile proxyFile = new(llvmBitcodeFile); + proxyFiles.Add(proxyFile); + aotAssembly.SetMetadata("LlvmBitcodeFile", proxyFile.TargetFile); if (parsedAotMode == MonoAotMode.LLVMOnlyInterp) { @@ -466,11 +466,11 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) if (parsedOutputType == MonoAotOutputType.AsmOnly) { aotArgs.Add("asmonly"); - aotArgs.Add($"llvm-outfile={tmpOutputFile}"); + aotArgs.Add($"llvm-outfile={proxyFile.TempFile}"); } else { - aotArgs.Add($"outfile={tmpOutputFile}"); + aotArgs.Add($"outfile={proxyFile.TempFile}"); } } else @@ -495,9 +495,10 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) case MonoAotOutputType.ObjectFile: { string objectFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.o")); - tmpOutputFile = objectFile + ".tmp"; - aotArgs.Add($"outfile={tmpOutputFile}"); - aotAssembly.SetMetadata("ObjectFile", objectFile); + ProxyFile proxyFile = new(objectFile); + proxyFiles.Add((proxyFile)); + aotArgs.Add($"outfile={proxyFile.TempFile}"); + aotAssembly.SetMetadata("ObjectFile", proxyFile.TargetFile); } break; @@ -506,9 +507,10 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) aotArgs.Add("asmonly"); string assemblerFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.s")); - tmpOutputFile = assemblerFile + ".tmp"; - aotArgs.Add($"outfile={tmpOutputFile}"); - aotAssembly.SetMetadata("AssemblerFile", assemblerFile); + ProxyFile proxyFile = new(assemblerFile); + proxyFiles.Add(proxyFile); + aotArgs.Add($"outfile={proxyFile.TempFile}"); + aotAssembly.SetMetadata("AssemblerFile", proxyFile.TargetFile); } break; @@ -522,10 +524,11 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) }; string libraryFileName = $"{LibraryFilePrefix}{assemblyFilename}{extension}"; string libraryFilePath = Path.Combine(OutputDir, libraryFileName); - tmpOutputFile = libraryFilePath + ".tmp"; + ProxyFile proxyFile = new(libraryFilePath); + proxyFiles.Add(proxyFile); - aotArgs.Add($"outfile={tmpOutputFile}"); - aotAssembly.SetMetadata("LibraryFile", libraryFilePath); + aotArgs.Add($"outfile={proxyFile.TempFile}"); + aotAssembly.SetMetadata("LibraryFile", proxyFile.TargetFile); } break; @@ -536,9 +539,10 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) if (UseLLVM) { string llvmObjectFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll-llvm.o")); - tmpOutputFile = llvmObjectFile; - aotArgs.Add($"llvm-outfile={tmpOutputFile}"); - aotAssembly.SetMetadata("LlvmObjectFile", llvmObjectFile); + ProxyFile proxyFile = new(llvmObjectFile); + proxyFiles.Add(proxyFile); + aotArgs.Add($"llvm-outfile={proxyFile.TempFile}"); + aotAssembly.SetMetadata("LlvmObjectFile", proxyFile.TargetFile); } } @@ -649,14 +653,27 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) return false; } - if (!File.Exists(tmpOutputFile)) + + bool copied = false; + foreach (var proxyFile in proxyFiles) { - Log.LogError($"Precompiling failed for {assembly}. Could not find output file {tmpOutputFile}"); - return false; + if (!File.Exists(proxyFile.TempFile)) + { + Log.LogError($"Precompiling failed for {assembly}. Could not find output file {proxyFile.TempFile}"); + return false; + } + + copied |= CopyOutputFileIfChanged(proxyFile); + File.Delete(proxyFile.TempFile); + } + + if (copied) + { + string copiedFiles = string.Join(", ", proxyFiles.Select(tf => Path.GetFileName(tf.TargetFile))); + int count = Interlocked.Increment(ref _numCompiled); + Log.LogMessage(MessageImportance.High, $"[{count}/{_totalNumAssemblies}] {Path.GetFileName(assembly)} -> {copiedFiles}"); } - CopyOutputFileIfChanged(assembly, tmpOutputFile); - File.Delete(tmpOutputFile); File.Delete(responseFilePath); compiledAssemblies.GetOrAdd(aotAssembly.ItemSpec, aotAssembly); @@ -699,33 +716,22 @@ private void PrecompileLibraryParallel(ITaskItem assemblyItem, string? monoPaths } } - private void CopyOutputFileIfChanged(string assemblyPath, string tmpOutputFile) + private bool CopyOutputFileIfChanged(ProxyFile proxyFile) { - if (Path.GetExtension(tmpOutputFile) != ".tmp") - { - Log.LogWarning($"Bug: expected tmpOutputFile to end in .tmp: {tmpOutputFile}"); - return; - } - - string finalOutputFile = Path.Combine(Path.GetDirectoryName(tmpOutputFile) ?? string.Empty, - Path.GetFileNameWithoutExtension(tmpOutputFile)); - if (!ShouldCopy(out string? cause)) { - Log.LogMessage(MessageImportance.Low, $"Skipping copying over {finalOutputFile} as the contents are unchanged"); - return; + Log.LogMessage(MessageImportance.Low, $"Skipping copying over {proxyFile.TargetFile} as the contents are unchanged"); + return false; } - if (File.Exists(finalOutputFile)) - File.Delete(finalOutputFile); + if (File.Exists(proxyFile.TargetFile)) + File.Delete(proxyFile.TargetFile); - File.Copy(tmpOutputFile, finalOutputFile); + File.Copy(proxyFile.TempFile, proxyFile.TargetFile); - Log.LogMessage(MessageImportance.Low, $"Copying {tmpOutputFile} to {finalOutputFile} because {cause}"); - _fileWrites.Add(finalOutputFile); - - int count = Interlocked.Increment(ref _numCompiled); - Log.LogMessage(MessageImportance.High, $"[{count}/{_totalNumAssemblies}] {Path.GetFileName(assemblyPath)} -> {Path.GetFileName(finalOutputFile)}"); + Log.LogMessage(MessageImportance.Low, $"Copying {proxyFile.TempFile} to {proxyFile.TargetFile} because {cause}"); + _fileWrites.Add(proxyFile.TargetFile); + return true; bool ShouldCopy([NotNullWhen(true)] out string? cause) { @@ -738,18 +744,18 @@ bool ShouldCopy([NotNullWhen(true)] out string? cause) return true; } - string newHash = Utils.ComputeHash(tmpOutputFile); - _newCache!.FileHashes[finalOutputFile] = newHash; + string newHash = Utils.ComputeHash(proxyFile.TempFile); + _newCache!.FileHashes[proxyFile.TargetFile] = newHash; - if (!File.Exists(finalOutputFile)) + if (!File.Exists(proxyFile.TargetFile)) { cause = $"the output file didn't exist"; return true; } string? oldHash; - if (!_oldCache!.FileHashes.TryGetValue(finalOutputFile, out oldHash)) - oldHash = Utils.ComputeHash(finalOutputFile); + if (!_oldCache!.FileHashes.TryGetValue(proxyFile.TargetFile, out oldHash)) + oldHash = Utils.ComputeHash(proxyFile.TargetFile); if (oldHash != newHash) { @@ -911,6 +917,18 @@ private IList ConvertAssembliesDictToOrderedList(ConcurrentDictionary } return outItems; } + + internal class ProxyFile + { + public ProxyFile(string targetFile) + { + this.TargetFile = targetFile; + this.TempFile = targetFile + ".tmp"; + } + + public string TargetFile { get; } + public string TempFile { get; } + } } public enum MonoAotMode @@ -946,6 +964,6 @@ public enum MonoAotModulesTableLanguage internal class CompilerCache { - [JsonPropertyName("assembly_hashes")] + [JsonPropertyName("file_hashes")] public ConcurrentDictionary FileHashes { get; set; } = new(); } From cc7742c67f033dcd47e287a49782b7d17a4508b1 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Thu, 12 Aug 2021 19:52:08 -0400 Subject: [PATCH 22/23] -bump sdk for workload testing --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 1db12f5321d298..be7f118c015936 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -164,7 +164,7 @@ 2.0.4 4.12.0 2.14.3 - 6.0.100-rc.1.21410.3 + 6.0.100-rc.1.21412.8 5.0.0-preview-20201009.2 From d11f69f1c6cb110f0d9933ba4516cf700f66c81c Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Sat, 14 Aug 2021 04:17:04 -0400 Subject: [PATCH 23/23] MonoAOTCompiler: don't use tmp files at all, when cache isn't being .. used. --- src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 205 ++++++++++-------- src/tasks/Common/Utils.cs | 2 +- .../WasmAppBuilder/IcallTableGenerator.cs | 2 +- .../WasmAppBuilder/PInvokeTableGenerator.cs | 2 +- src/tasks/WasmAppBuilder/WasmAppBuilder.cs | 2 +- 5 files changed, 113 insertions(+), 100 deletions(-) diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index 0820f334048999..6d0d5eeb1a49ba 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -203,9 +203,7 @@ public class MonoAOTCompiler : Microsoft.Build.Utilities.Task private MonoAotLibraryFormat parsedLibraryFormat; private MonoAotModulesTableLanguage parsedAotModulesTableLanguage; - private CompilerCache? _newCache; - private CompilerCache? _oldCache; - private bool _useCache => _newCache != null && _oldCache != null; + private FileCache? _cache; private int _numCompiled; private int _totalNumAssemblies; @@ -327,7 +325,7 @@ public override bool Execute() if (AdditionalAssemblySearchPaths != null) monoPaths = string.Join(Path.PathSeparator.ToString(), AdditionalAssemblySearchPaths); - InitCache(); + _cache = new FileCache(CacheFilePath, Log); //FIXME: check the nothing changed at all case @@ -364,12 +362,8 @@ public override bool Execute() if (numUnchanged > 0 && numUnchanged != _totalNumAssemblies) Log.LogMessage(MessageImportance.High, $"[{numUnchanged}/{_totalNumAssemblies}] skipped unchanged assemblies."); - if (_useCache) - { - var json = JsonSerializer.Serialize (_newCache, new JsonSerializerOptions { WriteIndented = true }); - File.WriteAllText(CacheFilePath!, json); + if (_cache.Save(CacheFilePath!)) _fileWrites.Add(CacheFilePath!); - } CompiledAssemblies = ConvertAssembliesDictToOrderedList(compiledAssemblies, Assemblies).ToArray(); FileWrites = _fileWrites.ToArray(); @@ -454,7 +448,7 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) aotArgs.Add("llvmonly"); string llvmBitcodeFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.bc")); - ProxyFile proxyFile = new(llvmBitcodeFile); + ProxyFile proxyFile = _cache!.NewFile(llvmBitcodeFile); proxyFiles.Add(proxyFile); aotAssembly.SetMetadata("LlvmBitcodeFile", proxyFile.TargetFile); @@ -495,7 +489,7 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) case MonoAotOutputType.ObjectFile: { string objectFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.o")); - ProxyFile proxyFile = new(objectFile); + ProxyFile proxyFile = _cache!.NewFile(objectFile); proxyFiles.Add((proxyFile)); aotArgs.Add($"outfile={proxyFile.TempFile}"); aotAssembly.SetMetadata("ObjectFile", proxyFile.TargetFile); @@ -507,7 +501,7 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) aotArgs.Add("asmonly"); string assemblerFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.s")); - ProxyFile proxyFile = new(assemblerFile); + ProxyFile proxyFile = _cache!.NewFile(assemblerFile); proxyFiles.Add(proxyFile); aotArgs.Add($"outfile={proxyFile.TempFile}"); aotAssembly.SetMetadata("AssemblerFile", proxyFile.TargetFile); @@ -524,7 +518,7 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) }; string libraryFileName = $"{LibraryFilePrefix}{assemblyFilename}{extension}"; string libraryFilePath = Path.Combine(OutputDir, libraryFileName); - ProxyFile proxyFile = new(libraryFilePath); + ProxyFile proxyFile = _cache!.NewFile(libraryFilePath); proxyFiles.Add(proxyFile); aotArgs.Add($"outfile={proxyFile.TempFile}"); @@ -539,7 +533,7 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) if (UseLLVM) { string llvmObjectFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll-llvm.o")); - ProxyFile proxyFile = new(llvmObjectFile); + ProxyFile proxyFile = _cache.NewFile(llvmObjectFile); proxyFiles.Add(proxyFile); aotArgs.Add($"llvm-outfile={proxyFile.TempFile}"); aotAssembly.SetMetadata("LlvmObjectFile", proxyFile.TargetFile); @@ -663,8 +657,8 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) return false; } - copied |= CopyOutputFileIfChanged(proxyFile); - File.Delete(proxyFile.TempFile); + copied |= proxyFile.CopyOutputFileIfChanged(); + _fileWrites.Add(proxyFile.TargetFile); } if (copied) @@ -675,7 +669,6 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) } File.Delete(responseFilePath); - compiledAssemblies.GetOrAdd(aotAssembly.ItemSpec, aotAssembly); return true; } @@ -716,57 +709,6 @@ private void PrecompileLibraryParallel(ITaskItem assemblyItem, string? monoPaths } } - private bool CopyOutputFileIfChanged(ProxyFile proxyFile) - { - if (!ShouldCopy(out string? cause)) - { - Log.LogMessage(MessageImportance.Low, $"Skipping copying over {proxyFile.TargetFile} as the contents are unchanged"); - return false; - } - - if (File.Exists(proxyFile.TargetFile)) - File.Delete(proxyFile.TargetFile); - - File.Copy(proxyFile.TempFile, proxyFile.TargetFile); - - Log.LogMessage(MessageImportance.Low, $"Copying {proxyFile.TempFile} to {proxyFile.TargetFile} because {cause}"); - _fileWrites.Add(proxyFile.TargetFile); - return true; - - bool ShouldCopy([NotNullWhen(true)] out string? cause) - { - cause = null; - - if (!_useCache) - { - // cache disabled - cause = "cache is disabled"; - return true; - } - - string newHash = Utils.ComputeHash(proxyFile.TempFile); - _newCache!.FileHashes[proxyFile.TargetFile] = newHash; - - if (!File.Exists(proxyFile.TargetFile)) - { - cause = $"the output file didn't exist"; - return true; - } - - string? oldHash; - if (!_oldCache!.FileHashes.TryGetValue(proxyFile.TargetFile, out oldHash)) - oldHash = Utils.ComputeHash(proxyFile.TargetFile); - - if (oldHash != newHash) - { - cause = $"hash for the file changed"; - return true; - } - - return false; - } - } - private bool GenerateAotModulesTable(ITaskItem[] assemblies, string[]? profilers, string outputFile) { var symbols = new List(); @@ -782,7 +724,7 @@ private bool GenerateAotModulesTable(ITaskItem[] assemblies, string[]? profilers if (!TryGetAssemblyName(asmPath, out string? assemblyName)) return false; - string symbolName = assemblyName.Replace ('.', '_').Replace ('-', '_'); + string symbolName = assemblyName.Replace ('.', '_').Replace ('-', '_').Replace(' ', '_'); symbols.Add($"mono_aot_module_{symbolName}_info"); } @@ -850,7 +792,7 @@ private bool GenerateAotModulesTable(ITaskItem[] assemblies, string[]? profilers } } - if (Utils.CopyIfDifferent(tmpAotModulesTablePath, outputFile)) + if (Utils.CopyIfDifferent(tmpAotModulesTablePath, outputFile, useHash: false)) { _fileWrites.Add(outputFile); Log.LogMessage(MessageImportance.Low, $"Generated {outputFile}"); @@ -859,26 +801,6 @@ private bool GenerateAotModulesTable(ITaskItem[] assemblies, string[]? profilers return true; } - private bool InitCache() - { - if (string.IsNullOrEmpty(CacheFilePath)) - { - Log.LogMessage(MessageImportance.Low, $"Disabling cache, because {nameof(CacheFilePath)} is not set"); - return false; - } - - if (File.Exists(CacheFilePath)) - { - _oldCache = (CompilerCache?)JsonSerializer.Deserialize(File.ReadAllText(CacheFilePath!), - typeof(CompilerCache), - new JsonSerializerOptions()); - } - - _oldCache ??= new(); - _newCache = new(); - return true; - } - private bool TryGetAssemblyName(string asmPath, [NotNullWhen(true)] out string? assemblyName) { assemblyName = null; @@ -917,17 +839,108 @@ private IList ConvertAssembliesDictToOrderedList(ConcurrentDictionary } return outItems; } +} + +internal class FileCache +{ + private CompilerCache? _newCache; + private CompilerCache? _oldCache; + + public bool Enabled { get; } + public TaskLoggingHelper Log { get; } + + public FileCache(string? cacheFilePath, TaskLoggingHelper log) + { + Log = log; + if (string.IsNullOrEmpty(cacheFilePath)) + { + Log.LogMessage(MessageImportance.Low, $"Disabling cache, because CacheFilePath is not set"); + return; + } + + Enabled = true; + if (File.Exists(cacheFilePath)) + { + _oldCache = (CompilerCache?)JsonSerializer.Deserialize(File.ReadAllText(cacheFilePath), + typeof(CompilerCache), + new JsonSerializerOptions()); + } + + _oldCache ??= new(); + _newCache = new(); + } - internal class ProxyFile + public bool ShouldCopy(ProxyFile proxyFile, [NotNullWhen(true)] out string? cause) { - public ProxyFile(string targetFile) + cause = null; + + string newHash = Utils.ComputeHash(proxyFile.TempFile); + _newCache!.FileHashes[proxyFile.TargetFile] = newHash; + + if (!File.Exists(proxyFile.TargetFile)) + { + cause = $"the output file didn't exist"; + return true; + } + + string? oldHash; + if (!_oldCache!.FileHashes.TryGetValue(proxyFile.TargetFile, out oldHash)) + oldHash = Utils.ComputeHash(proxyFile.TargetFile); + + if (oldHash != newHash) { - this.TargetFile = targetFile; - this.TempFile = targetFile + ".tmp"; + cause = $"hash for the file changed"; + return true; } - public string TargetFile { get; } - public string TempFile { get; } + return false; + } + + public bool Save(string? cacheFilePath) + { + if (!Enabled || string.IsNullOrEmpty(cacheFilePath)) + return false; + + var json = JsonSerializer.Serialize (_newCache, new JsonSerializerOptions { WriteIndented = true }); + File.WriteAllText(cacheFilePath!, json); + return true; + } + + public ProxyFile NewFile(string targetFile) => new ProxyFile(targetFile, this); +} + +internal class ProxyFile +{ + public string TargetFile { get; } + public string TempFile { get; } + private FileCache _cache; + + public ProxyFile(string targetFile, FileCache cache) + { + _cache = cache; + this.TargetFile = targetFile; + this.TempFile = _cache.Enabled ? targetFile + ".tmp" : targetFile; + } + + public bool CopyOutputFileIfChanged() + { + if (!_cache.Enabled) + return true; + + if (!_cache.ShouldCopy(this, out string? cause)) + { + _cache.Log.LogMessage(MessageImportance.Low, $"Skipping copying over {TargetFile} as the contents are unchanged"); + return false; + } + + if (File.Exists(TargetFile)) + File.Delete(TargetFile); + + File.Copy(TempFile, TargetFile); + File.Delete(TempFile); + + _cache.Log.LogMessage(MessageImportance.Low, $"Copying {TempFile} to {TargetFile} because {cause}"); + return true; } } diff --git a/src/tasks/Common/Utils.cs b/src/tasks/Common/Utils.cs index 5887ea18fa37b9..2c1a8b49370c7a 100644 --- a/src/tasks/Common/Utils.cs +++ b/src/tasks/Common/Utils.cs @@ -206,7 +206,7 @@ internal static string CreateTemporaryBatchFile(string command) return file; } - public static bool CopyIfDifferent(string src, string dst, bool useHash=false) + public static bool CopyIfDifferent(string src, string dst, bool useHash) { if (!File.Exists(src)) throw new ArgumentException($"Cannot find {src} file to copy", nameof(src)); diff --git a/src/tasks/WasmAppBuilder/IcallTableGenerator.cs b/src/tasks/WasmAppBuilder/IcallTableGenerator.cs index 592ac56a3e1233..13c75c39bc64cd 100644 --- a/src/tasks/WasmAppBuilder/IcallTableGenerator.cs +++ b/src/tasks/WasmAppBuilder/IcallTableGenerator.cs @@ -54,7 +54,7 @@ public void GenIcallTable(string runtimeIcallTableFile, string[] assemblies) using (var w = File.CreateText(tmpFileName)) EmitTable (w); - if (Utils.CopyIfDifferent(tmpFileName, OutputPath)) + if (Utils.CopyIfDifferent(tmpFileName, OutputPath, useHash: false)) Log.LogMessage(MessageImportance.Low, $"Generating icall table to '{OutputPath}'."); else Log.LogMessage(MessageImportance.Low, $"Icall table in {OutputPath} is unchanged."); diff --git a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs index d5a365f55c0285..f6b89533f98632 100644 --- a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs +++ b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs @@ -57,7 +57,7 @@ public void GenPInvokeTable(string[] pinvokeModules, string[] assemblies) EmitNativeToInterp(w, callbacks); } - if (Utils.CopyIfDifferent(tmpFileName, OutputPath)) + if (Utils.CopyIfDifferent(tmpFileName, OutputPath, useHash: false)) Log.LogMessage(MessageImportance.Low, $"Generating pinvoke table to '{OutputPath}'."); else Log.LogMessage(MessageImportance.Low, $"PInvoke table in {OutputPath} is unchanged."); diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs index fd8f489e1c861c..57cb7c383f107a 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs +++ b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs @@ -255,7 +255,7 @@ public override bool Execute () sw.Write(json); } string monoConfigPath = Path.Combine(AppDir, "mono-config.json"); - Utils.CopyIfDifferent(tmpMonoConfigPath, monoConfigPath); + Utils.CopyIfDifferent(tmpMonoConfigPath, monoConfigPath, useHash: false); _fileWrites.Add(monoConfigPath); if (ExtraFilesToDeploy != null)