diff --git a/eng/Versions.props b/eng/Versions.props index 53d03a6553dc2e..807a903cfac3ca 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -165,7 +165,7 @@ 2.0.4 4.12.0 2.14.3 - 6.0.100-rc.1.21402.6 + 6.0.100-rc.1.21412.8 5.0.0-preview-20201009.2 diff --git a/eng/testing/scenarios/BuildWasmAppsJobsList.txt b/eng/testing/scenarios/BuildWasmAppsJobsList.txt new file mode 100644 index 00000000000000..ba322274dfe62c --- /dev/null +++ b/eng/testing/scenarios/BuildWasmAppsJobsList.txt @@ -0,0 +1,14 @@ +BlazorWasmTests +FlagsChangeRebuildTest +InvariantGlobalizationTests +LocalEMSDKTests +MainWithArgsTests +NativeBuildTests +NativeLibraryTests +NoopNativeRebuildTest +RebuildTests +ReferenceNewAssemblyRebuildTest +SatelliteAssembliesTests +SimpleSourceChangeRebuildTest +WasmBuildAppTest +WorkloadTests diff --git a/eng/testing/tests.wasm.targets b/eng/testing/tests.wasm.targets index 8d6829e47b3387..386ef21d204f59 100644 --- a/eng/testing/tests.wasm.targets +++ b/eng/testing/tests.wasm.targets @@ -21,7 +21,7 @@ <_XHarnessArgs Condition="'$(OS)' != 'Windows_NT'">wasm $XHARNESS_COMMAND --app=. --output-directory=$XHARNESS_OUT <_XHarnessArgs Condition="'$(OS)' == 'Windows_NT'">wasm %XHARNESS_COMMAND% --app=. --output-directory=%XHARNESS_OUT% - <_XHarnessArgs Condition="'$(Scenario)' != 'WasmTestOnBrowser'">$(_XHarnessArgs) --engine=$(JSEngine) $(JSEngineArgs) --js-file=runtime.js + <_XHarnessArgs Condition="'$(Scenario)' != 'WasmTestOnBrowser' and '$(Scenario)' != 'BuildWasmApps'">$(_XHarnessArgs) --engine=$(JSEngine) $(JSEngineArgs) --js-file=runtime.js <_XHarnessArgs Condition="'$(BrowserHost)' == 'windows'">$(_XHarnessArgs) --browser=chrome --browser-path=%HELIX_CORRELATION_PAYLOAD%\chrome-win\chrome.exe <_XHarnessArgs Condition="'$(IsFunctionalTest)' == 'true'" >$(_XHarnessArgs) --expected-exit-code=$(ExpectedExitCode) <_XHarnessArgs Condition="'$(WasmXHarnessArgs)' != ''" >$(_XHarnessArgs) $(WasmXHarnessArgs) diff --git a/src/libraries/sendtohelixhelp.proj b/src/libraries/sendtohelixhelp.proj index b106d310e02761..14cf90630710d4 100644 --- a/src/libraries/sendtohelixhelp.proj +++ b/src/libraries/sendtohelixhelp.proj @@ -53,6 +53,8 @@ $(TestRunNamePrefix)$(TestRunNamePrefixSuffix)- $(TestRunNamePrefix)$(Scenario)- + $(RepositoryEngineeringDir)\testing\scenarios\BuildWasmAppsJobsList.txt + $(WaitForWorkItemCompletion) $(RepoRoot)src\mono\wasm\emsdk\ @@ -385,6 +387,10 @@ + + + + <_WorkItem Include="$(TestArchiveRoot)browseronly/**/*.zip" Condition="'$(TargetOS)' == 'Browser' and '$(Scenario)' == 'WasmTestOnBrowser'" /> - + %(Identity) $(HelixCommand) $(_workItemTimeout) + + <_BuildWasmAppsPayloadArchive>@(_WorkItem) + + + + + $(_BuildWasmAppsPayloadArchive) + set "HELIX_XUNIT_ARGS=-class Wasm.Build.Tests.%(Identity)" + export "HELIX_XUNIT_ARGS=-class Wasm.Build.Tests.%(Identity)" + $(HelixCommand) + $(_workItemTimeout) + + + dotnet exec $XHARNESS_CLI_PATH $HELIX_WORKITEM_UPLOAD_ROOT/xharness-output diff --git a/src/mono/wasm/build/WasmApp.Native.targets b/src/mono/wasm/build/WasmApp.Native.targets index 3e61a7074d3fb4..5b43374ab16b7f 100644 --- a/src/mono/wasm/build/WasmApp.Native.targets +++ b/src/mono/wasm/build/WasmApp.Native.targets @@ -33,6 +33,10 @@ true + + <_MonoComponent Include="hot_reload;debugger" /> + + @@ -145,6 +149,10 @@ <_WasmICallTablePath>$(_WasmIntermediateOutputPath)icall-table.h <_WasmRuntimeICallTablePath>$(_WasmIntermediateOutputPath)runtime-icall-table.h <_WasmPInvokeTablePath>$(_WasmIntermediateOutputPath)pinvoke-table.h + <_WasmPInvokeHPath>$(_WasmRuntimePackIncludeDir)wasm\pinvoke.h + <_DriverGenCPath>$(_WasmIntermediateOutputPath)driver-gen.c + + <_DriverGenCNeeded Condition="'$(_DriverGenCNeeded)' == '' and '$(RunAOTCompilation)' == 'true'">true <_EmccAssertionLevelDefault>0 <_EmccOptimizationFlagDefault Condition="'$(_WasmDevel)' == 'true'">-O0 -s ASSERTIONS=$(_EmccAssertionLevelDefault) @@ -159,10 +167,15 @@ <_EmccCompileOutputMessageImportance Condition="'$(EmccVerbose)' == 'true'">Normal <_EmccCompileOutputMessageImportance Condition="'$(EmccVerbose)' != 'true'">Low + <_EmccCompileBitcodeRsp>$(_WasmIntermediateOutputPath)emcc-compile-bc.rsp + <_EmccLinkRsp>$(_WasmIntermediateOutputPath)emcc-link.rsp + 536870912 + <_WasmLinkDependencies Remove="@(_WasmLinkDependencies)" /> + <_EmccCommonFlags Include="$(_DefaultEmccFlags)" /> <_EmccCommonFlags Include="$(EmccFlags)" /> <_EmccCommonFlags Include="-s DISABLE_EXCEPTION_CATCHING=0" /> @@ -188,9 +201,20 @@ <_EmccCFlags Include=""-I%(_EmccIncludePaths.Identity)"" /> <_EmccCFlags Include="-g" Condition="'$(WasmNativeDebugSymbols)' == 'true'" /> - <_EmccCFlags Include="$(EmccExtraCFlags)" /> + + <_EmccLDFlags Include="$(EmccLinkOptimizationFlag)" /> + <_EmccLDFlags Include="@(_EmccCommonFlags)" /> + <_EmccLDFlags Include="-s TOTAL_MEMORY=$(EmccTotalMemory)" /> + + <_DriverCDependencies Include="$(_WasmPInvokeHPath);$(_WasmICallTablePath)" /> + <_DriverCDependencies Include="$(_DriverGenCPath)" Condition="'$(_DriverGenCNeeded)' == 'true'" /> + + <_WasmRuntimePackSrcFile Include="$(_WasmRuntimePackSrcDir)pinvoke.c" + Dependencies="$(_WasmPInvokeHPath);$(_WasmPInvokeTablePath)" /> + <_WasmRuntimePackSrcFile Include="$(_WasmRuntimePackSrcDir)driver.c" + Dependencies="@(_DriverCDependencies)" /> + <_WasmRuntimePackSrcFile Include="$(_WasmRuntimePackSrcDir)corebindings.c" /> - <_WasmRuntimePackSrcFile Include="$(_WasmRuntimePackSrcDir)*.c" /> <_WasmRuntimePackSrcFile ObjectFile="$(_WasmIntermediateOutputPath)%(FileName).o" /> <_DotnetJSSrcFile Include="$(_WasmRuntimePackSrcDir)\*.js" /> @@ -226,62 +250,64 @@ OutputPath="$(_WasmICallTablePath)" /> - - - <_WasmSourceFileToCompile Remove="@(_WasmSourceFileToCompile)" /> - <_WasmSourceFileToCompile Include="@(_WasmRuntimePackSrcFile)" /> - + + <_EmBuilder Condition="$([MSBuild]::IsOSPlatform('WINDOWS'))">embuilder.bat <_EmBuilder Condition="!$([MSBuild]::IsOSPlatform('WINDOWS'))">embuilder.py + + <_EmccCFlags Include="$(EmccExtraCFlags)" /> + + + + <_WasmSourceFileToCompile Remove="@(_WasmSourceFileToCompile)" /> + <_WasmSourceFileToCompile Include="@(_WasmRuntimePackSrcFile)" Dependencies="%(_WasmRuntimePackSrcFile.Dependencies);$(_EmccDefaultFlagsRsp);$(_EmccCompileRsp)" /> + - - - - - - <_MonoComponent Include="hot_reload;debugger" /> - - - - + - - - <_EmccLDFlags Include="$(EmccLinkOptimizationFlag)" /> - <_EmccLDFlags Include="@(_EmccCommonFlags)" /> - - <_EmccLDFlags Include="-s TOTAL_MEMORY=$(EmccTotalMemory)" /> - <_EmccLDFlags Include="$(EmccExtraLDFlags)" /> + <_BitCodeFile Dependencies="%(_BitCodeFile.Dependencies);$(_EmccDefaultFlagsRsp);$(_EmccCompileBitcodeRsp)" /> + + + - + <_BitcodeLDFlags Include="@(_EmccLDFlags)" /> + <_BitcodeLDFlags Include="$(EmccExtraBitcodeLDFlags)" /> + + + + + + + <_WasmNativeFileForLinking Include="%(_BitcodeFile.ObjectFile)" /> <_WasmNativeFileForLinking Include="%(_WasmSourceFileToCompile.ObjectFile)" /> @@ -293,25 +319,35 @@ <_EmccLinkStepArgs Include="@(_EmccLDFlags)" /> <_EmccLinkStepArgs Include="--js-library "%(_DotnetJSSrcFile.Identity)"" /> - <_EmccLinkStepArgs Include="--js-library "%(_WasmExtraJSFile.Identity)"" Condition="'%(_WasmExtraJSFile.Kind)' == 'js-library'" /> + <_WasmLinkDependencies Include="@(_DotnetJSSrcFile)" /> + <_EmccLinkStepArgs Include="--js-library "%(_WasmExtraJSFile.Identity)"" Condition="'%(_WasmExtraJSFile.Kind)' == 'js-library'" /> <_EmccLinkStepArgs Include="--pre-js "%(_WasmExtraJSFile.Identity)"" Condition="'%(_WasmExtraJSFile.Kind)' == 'pre-js'" /> <_EmccLinkStepArgs Include="--post-js "%(_WasmExtraJSFile.Identity)"" Condition="'%(_WasmExtraJSFile.Kind)' == 'post-js'" /> + <_WasmLinkDependencies Include="@(_WasmExtraJSFile)" Condition="'%(_WasmExtraJSFile.Kind)' == 'js-library' or '%(_WasmExtraJSFile.Kind)' == 'pre-js' or '%(_WasmExtraJSFile.Kind)' == 'post-js'" /> <_EmccLinkStepArgs Include=""%(_WasmNativeFileForLinking.Identity)"" /> + <_WasmLinkDependencies Include="@(_WasmNativeFileForLinking)" /> + <_EmccLinkStepArgs Include="-o "$(_WasmIntermediateOutputPath)dotnet.js"" /> - + <_WasmLinkDependencies Include="$(_EmccLinkRsp)" /> - - <_EmccLinkRsp>$(_WasmIntermediateOutputPath)emcc-link.rsp - + <_EmccLinkStepArgs Include="$(EmccExtraLDFlags)" /> + + + + - + + @@ -325,16 +361,15 @@ $(EmccExtraCFlags) -DDRIVER_GEN=1 + <_DriverGenCNeeded>true void mono_profiler_init_aot (const char *desc)%3B EMSCRIPTEN_KEEPALIVE void mono_wasm_load_profiler_aot (const char *desc) { mono_profiler_init_aot (desc)%3B } - - <_DriverGenCPath>$(_WasmIntermediateOutputPath)driver-gen.c - + @@ -414,6 +449,7 @@ EMSCRIPTEN_KEEPALIVE void mono_wasm_load_profiler_aot (const char *desc) { mono_ LLVMOnlyInterp + <_AOTCompilerCacheFile>$(_WasmIntermediateOutputPath)aot_compiler_cache.json diff --git a/src/mono/wasm/build/WasmApp.targets b/src/mono/wasm/build/WasmApp.targets index 32c55ee12bf6b0..e5ceb13def292e 100644 --- a/src/mono/wasm/build/WasmApp.targets +++ b/src/mono/wasm/build/WasmApp.targets @@ -107,6 +107,8 @@ <_WasmRuntimePackSrcDir>$([MSBuild]::NormalizeDirectory($(MicrosoftNetCoreAppRuntimePackRidNativeDir), 'src')) <_WasmIntermediateOutputPath>$([MSBuild]::NormalizeDirectory($(IntermediateOutputPath), 'wasm')) + + <_DriverGenCPath>$(_WasmIntermediateOutputPath)driver-gen.c @@ -125,10 +127,11 @@ <_MainAssemblyPath Condition="'%(WasmAssembliesToBundle.FileName)' == $(AssemblyName) and '%(WasmAssembliesToBundle.Extension)' == '.dll' and $(WasmGenerateAppBundle) == 'true'">%(WasmAssembliesToBundle.Identity) <_WasmRuntimeConfigFilePath Condition="$(_MainAssemblyPath) != ''">$([System.IO.Path]::ChangeExtension($(_MainAssemblyPath), '.runtimeconfig.json')) + <_ParsedRuntimeConfigFilePath Condition="'$(_MainAssemblyPath)' != ''">$([System.IO.Path]::GetDirectoryName($(_MainAssemblyPath)))\runtimeconfig.bin - @@ -143,17 +146,15 @@ - - - <_ParsedRuntimeConfigFilePath>$([System.IO.Path]::GetDirectoryName($(_MainAssemblyPath)))\runtimeconfig.bin - - + <_RuntimeConfigReservedProperties Include="RUNTIME_IDENTIFIER"/> <_RuntimeConfigReservedProperties Include="APP_CONTEXT_BASE_DIRECTORY"/> - public string? LLVMDebug { get; set; } = "nodebug"; + /// + /// File used to track hashes of assemblies, to act as a cache + /// Output files don't get written, if they haven't changed + /// + public string? CacheFilePath { get; set; } + [Output] public string[]? FileWrites { get; private set; } private List _fileWrites = new(); - private ConcurrentBag compiledAssemblies = new ConcurrentBag(); + private ConcurrentDictionary compiledAssemblies = new(); + private MonoAotMode parsedAotMode; private MonoAotOutputType parsedOutputType; private MonoAotLibraryFormat parsedLibraryFormat; private MonoAotModulesTableLanguage parsedAotModulesTableLanguage; + private FileCache? _cache; + private int _numCompiled; + private int _totalNumAssemblies; + public override bool Execute() { - if (string.IsNullOrEmpty(CompilerBinaryPath)) - { - throw new ArgumentException($"'{nameof(CompilerBinaryPath)}' is required.", nameof(CompilerBinaryPath)); - } - if (!File.Exists(CompilerBinaryPath)) { - throw new ArgumentException($"'{CompilerBinaryPath}' doesn't exist.", nameof(CompilerBinaryPath)); + Log.LogError($"{nameof(CompilerBinaryPath)}='{CompilerBinaryPath}' doesn't exist."); + return false; } if (Assemblies.Length == 0) { - throw new ArgumentException($"'{nameof(Assemblies)}' is required.", nameof(Assemblies)); + Log.LogError($"'{nameof(Assemblies)}' is required."); + return false; } if (!Path.IsPathRooted(OutputDir)) @@ -291,7 +302,7 @@ public override bool Execute() // AOT modules for static linking, needs the aot modules table UseStaticLinking = true; - if (!GenerateAotModulesTable(Assemblies, Profilers)) + if (!GenerateAotModulesTable(Assemblies, Profilers, AotModulesTablePath)) return false; } @@ -314,22 +325,47 @@ public override bool Execute() if (AdditionalAssemblySearchPaths != null) monoPaths = string.Join(Path.PathSeparator.ToString(), AdditionalAssemblySearchPaths); - if (DisableParallelAot) + _cache = new FileCache(CacheFilePath, Log); + + //FIXME: check the nothing changed at all case + + _totalNumAssemblies = Assemblies.Length; + int allowedParallelism = Math.Min(Assemblies.Length, Environment.ProcessorCount); + if (BuildEngine is IBuildEngine9 be9) + allowedParallelism = be9.RequestCores(allowedParallelism); + + if (DisableParallelAot || allowedParallelism == 1) { foreach (var assemblyItem in Assemblies) { - if (!PrecompileLibrary(assemblyItem, monoPaths)) + if (!PrecompileLibrarySerial(assemblyItem, monoPaths)) return !Log.HasLoggedErrors; } } else { - Parallel.ForEach(Assemblies, - new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }, - assemblyItem => PrecompileLibrary(assemblyItem, monoPaths)); + ParallelLoopResult result = Parallel.ForEach( + Assemblies, + new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism }, + (assemblyItem, state) => PrecompileLibraryParallel(assemblyItem, monoPaths, state)); + + if (!result.IsCompleted) + { + if (!Log.HasLoggedErrors) + Log.LogError("Unknown failed occured while compiling"); + + return false; + } } - CompiledAssemblies = compiledAssemblies.ToArray(); + int numUnchanged = _totalNumAssemblies - _numCompiled; + if (numUnchanged > 0 && numUnchanged != _totalNumAssemblies) + Log.LogMessage(MessageImportance.High, $"[{numUnchanged}/{_totalNumAssemblies}] skipped unchanged assemblies."); + + if (_cache.Save(CacheFilePath!)) + _fileWrites.Add(CacheFilePath!); + + CompiledAssemblies = ConvertAssembliesDictToOrderedList(compiledAssemblies, Assemblies).ToArray(); FileWrites = _fileWrites.ToArray(); return !Log.HasLoggedErrors; @@ -343,6 +379,7 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) var aotArgs = new List(); var processArgs = new List(); bool isDedup = assembly == DedupAssembly; + List proxyFiles = new(capacity: 5); string msgPrefix = $"[{Path.GetFileName(assembly)}] "; var a = assemblyItem.GetMetadata("AotArguments"); @@ -357,8 +394,6 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) processArgs.AddRange(p.Split(new char[]{ ';' }, StringSplitOptions.RemoveEmptyEntries)); } - Log.LogMessage(MessageImportance.Low, $"[AOT] {assembly}"); - processArgs.Add("--debug"); // add LLVM options @@ -413,7 +448,9 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) aotArgs.Add("llvmonly"); string llvmBitcodeFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.bc")); - aotAssembly.SetMetadata("LlvmBitcodeFile", llvmBitcodeFile); + ProxyFile proxyFile = _cache!.NewFile(llvmBitcodeFile); + proxyFiles.Add(proxyFile); + aotAssembly.SetMetadata("LlvmBitcodeFile", proxyFile.TargetFile); if (parsedAotMode == MonoAotMode.LLVMOnlyInterp) { @@ -423,11 +460,11 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) if (parsedOutputType == MonoAotOutputType.AsmOnly) { aotArgs.Add("asmonly"); - aotArgs.Add($"llvm-outfile={llvmBitcodeFile}"); + aotArgs.Add($"llvm-outfile={proxyFile.TempFile}"); } else { - aotArgs.Add($"outfile={llvmBitcodeFile}"); + aotArgs.Add($"outfile={proxyFile.TempFile}"); } } else @@ -447,40 +484,59 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) aotArgs.Add("interp"); } - if (parsedOutputType == MonoAotOutputType.ObjectFile) + switch (parsedOutputType) { - string objectFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.o")); - aotArgs.Add($"outfile={objectFile}"); - aotAssembly.SetMetadata("ObjectFile", objectFile); - } - else if (parsedOutputType == MonoAotOutputType.AsmOnly) - { - aotArgs.Add("asmonly"); + case MonoAotOutputType.ObjectFile: + { + string objectFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.o")); + ProxyFile proxyFile = _cache!.NewFile(objectFile); + proxyFiles.Add((proxyFile)); + aotArgs.Add($"outfile={proxyFile.TempFile}"); + aotAssembly.SetMetadata("ObjectFile", proxyFile.TargetFile); + } + break; - string assemblerFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.s")); - aotArgs.Add($"outfile={assemblerFile}"); - aotAssembly.SetMetadata("AssemblerFile", assemblerFile); - } - else if (parsedOutputType == MonoAotOutputType.Library) - { - string extension = parsedLibraryFormat switch { - MonoAotLibraryFormat.Dll => ".dll", - MonoAotLibraryFormat.Dylib => ".dylib", - MonoAotLibraryFormat.So => ".so", - _ => throw new ArgumentOutOfRangeException() - }; - string libraryFileName = $"{LibraryFilePrefix}{assemblyFilename}{extension}"; - string libraryFilePath = Path.Combine(OutputDir, libraryFileName); - - aotArgs.Add($"outfile={libraryFilePath}"); - aotAssembly.SetMetadata("LibraryFile", libraryFilePath); + case MonoAotOutputType.AsmOnly: + { + aotArgs.Add("asmonly"); + + string assemblerFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.s")); + ProxyFile proxyFile = _cache!.NewFile(assemblerFile); + proxyFiles.Add(proxyFile); + aotArgs.Add($"outfile={proxyFile.TempFile}"); + aotAssembly.SetMetadata("AssemblerFile", proxyFile.TargetFile); + } + break; + + case MonoAotOutputType.Library: + { + string extension = parsedLibraryFormat switch { + MonoAotLibraryFormat.Dll => ".dll", + MonoAotLibraryFormat.Dylib => ".dylib", + MonoAotLibraryFormat.So => ".so", + _ => throw new ArgumentOutOfRangeException() + }; + string libraryFileName = $"{LibraryFilePrefix}{assemblyFilename}{extension}"; + string libraryFilePath = Path.Combine(OutputDir, libraryFileName); + ProxyFile proxyFile = _cache!.NewFile(libraryFilePath); + proxyFiles.Add(proxyFile); + + aotArgs.Add($"outfile={proxyFile.TempFile}"); + aotAssembly.SetMetadata("LibraryFile", proxyFile.TargetFile); + } + break; + + default: + throw new Exception($"Bug: Unhandled MonoAotOutputType: {parsedAotMode}"); } if (UseLLVM) { string llvmObjectFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll-llvm.o")); - aotArgs.Add($"llvm-outfile={llvmObjectFile}"); - aotAssembly.SetMetadata("LlvmObjectFile", llvmObjectFile); + ProxyFile proxyFile = _cache.NewFile(llvmObjectFile); + proxyFiles.Add(proxyFile); + aotArgs.Add($"llvm-outfile={proxyFile.TempFile}"); + aotAssembly.SetMetadata("LlvmObjectFile", proxyFile.TargetFile); } } @@ -555,14 +611,6 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) string workingDir = assemblyDir; - // Log the command in a compact format which can be copy pasted - { - StringBuilder envStr = new StringBuilder(string.Empty); - foreach (KeyValuePair kvp in envVariables) - envStr.Append($"{kvp.Key}={kvp.Value} "); - Log.LogMessage(MessageImportance.Low, $"{msgPrefix}Exec (with response file contents expanded) in {workingDir}: {envStr}{CompilerBinaryPath} {responseFileContent}"); - } - try { // run the AOT compiler @@ -571,12 +619,24 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) $"--response=\"{responseFilePath}\"", envVariables, workingDir, - silent: false, + silent: true, debugMessageImportance: MessageImportance.Low, label: Path.GetFileName(assembly)); + + var importance = exitCode == 0 ? MessageImportance.Low : MessageImportance.High; + // Log the command in a compact format which can be copy pasted + { + StringBuilder envStr = new StringBuilder(string.Empty); + foreach (KeyValuePair kvp in envVariables) + envStr.Append($"{kvp.Key}={kvp.Value} "); + Log.LogMessage(importance, $"{msgPrefix}Exec (with response file contents expanded) in {workingDir}: {envStr}{CompilerBinaryPath} {responseFileContent}"); + } + + Log.LogMessage(importance, output); + if (exitCode != 0) { - Log.LogError($"Precompiling failed for {assembly}: {output}"); + Log.LogError($"Precompiling failed for {assembly}"); return false; } } @@ -587,13 +647,69 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) return false; } - File.Delete(responseFilePath); - compiledAssemblies.Add(aotAssembly); + bool copied = false; + foreach (var proxyFile in proxyFiles) + { + if (!File.Exists(proxyFile.TempFile)) + { + Log.LogError($"Precompiling failed for {assembly}. Could not find output file {proxyFile.TempFile}"); + return false; + } + + copied |= proxyFile.CopyOutputFileIfChanged(); + _fileWrites.Add(proxyFile.TargetFile); + } + + if (copied) + { + string copiedFiles = string.Join(", ", proxyFiles.Select(tf => Path.GetFileName(tf.TargetFile))); + int count = Interlocked.Increment(ref _numCompiled); + Log.LogMessage(MessageImportance.High, $"[{count}/{_totalNumAssemblies}] {Path.GetFileName(assembly)} -> {copiedFiles}"); + } + + File.Delete(responseFilePath); + compiledAssemblies.GetOrAdd(aotAssembly.ItemSpec, aotAssembly); return true; } - private bool GenerateAotModulesTable(ITaskItem[] assemblies, string[]? profilers) + private bool PrecompileLibrarySerial(ITaskItem assemblyItem, string? monoPaths) + { + try + { + if (!PrecompileLibrary(assemblyItem, monoPaths)) + return !Log.HasLoggedErrors; + return true; + } + catch (Exception ex) + { + if (Log.HasLoggedErrors) + Log.LogMessage(MessageImportance.Low, $"Precompile failed for {assemblyItem}: {ex}"); + else + Log.LogError($"Precompile failed for {assemblyItem}: {ex}"); + + return false; + } + } + + private void PrecompileLibraryParallel(ITaskItem assemblyItem, string? monoPaths, ParallelLoopState state) + { + try + { + if (!PrecompileLibrary(assemblyItem, monoPaths)) + state.Break(); + } + catch (Exception ex) + { + if (Log.HasLoggedErrors) + Log.LogMessage(MessageImportance.Low, $"Precompile failed for {assemblyItem}: {ex}"); + else + Log.LogError($"Precompile failed for {assemblyItem}: {ex}"); + state.Break(); + } + } + + private bool GenerateAotModulesTable(ITaskItem[] assemblies, string[]? profilers, string outputFile) { var symbols = new List(); foreach (var asm in assemblies) @@ -608,15 +724,15 @@ private bool GenerateAotModulesTable(ITaskItem[] assemblies, string[]? profilers if (!TryGetAssemblyName(asmPath, out string? assemblyName)) return false; - string symbolName = assemblyName.Replace ('.', '_').Replace ('-', '_'); + string symbolName = assemblyName.Replace ('.', '_').Replace ('-', '_').Replace(' ', '_'); symbols.Add($"mono_aot_module_{symbolName}_info"); } - Directory.CreateDirectory(Path.GetDirectoryName(AotModulesTablePath!)!); + Directory.CreateDirectory(Path.GetDirectoryName(outputFile)!); - using (var writer = File.CreateText(AotModulesTablePath!)) + string tmpAotModulesTablePath = Path.GetTempFileName(); + using (var writer = File.CreateText(tmpAotModulesTablePath)) { - _fileWrites.Add(AotModulesTablePath!); if (parsedAotModulesTableLanguage == MonoAotModulesTableLanguage.C) { writer.WriteLine("#include "); @@ -674,7 +790,12 @@ private bool GenerateAotModulesTable(ITaskItem[] assemblies, string[]? profilers { throw new NotSupportedException(); } - Log.LogMessage(MessageImportance.Low, $"Generated {AotModulesTablePath}"); + } + + if (Utils.CopyIfDifferent(tmpAotModulesTablePath, outputFile, useHash: false)) + { + _fileWrites.Add(outputFile); + Log.LogMessage(MessageImportance.Low, $"Generated {outputFile}"); } return true; @@ -705,6 +826,122 @@ private bool TryGetAssemblyName(string asmPath, [NotNullWhen(true)] out string? return false; } } + + private IList ConvertAssembliesDictToOrderedList(ConcurrentDictionary dict, ITaskItem[] items) + { + List outItems = new(items.Length); + foreach (ITaskItem item in items) + { + if (!dict.TryGetValue(item.ItemSpec, out ITaskItem? dictItem)) + throw new LogAsErrorException($"Bug: Could not find item in the dict with key {item.ItemSpec}"); + + outItems.Add(dictItem); + } + return outItems; + } +} + +internal class FileCache +{ + private CompilerCache? _newCache; + private CompilerCache? _oldCache; + + public bool Enabled { get; } + public TaskLoggingHelper Log { get; } + + public FileCache(string? cacheFilePath, TaskLoggingHelper log) + { + Log = log; + if (string.IsNullOrEmpty(cacheFilePath)) + { + Log.LogMessage(MessageImportance.Low, $"Disabling cache, because CacheFilePath is not set"); + return; + } + + Enabled = true; + if (File.Exists(cacheFilePath)) + { + _oldCache = (CompilerCache?)JsonSerializer.Deserialize(File.ReadAllText(cacheFilePath), + typeof(CompilerCache), + new JsonSerializerOptions()); + } + + _oldCache ??= new(); + _newCache = new(); + } + + public bool ShouldCopy(ProxyFile proxyFile, [NotNullWhen(true)] out string? cause) + { + cause = null; + + string newHash = Utils.ComputeHash(proxyFile.TempFile); + _newCache!.FileHashes[proxyFile.TargetFile] = newHash; + + if (!File.Exists(proxyFile.TargetFile)) + { + cause = $"the output file didn't exist"; + return true; + } + + string? oldHash; + if (!_oldCache!.FileHashes.TryGetValue(proxyFile.TargetFile, out oldHash)) + oldHash = Utils.ComputeHash(proxyFile.TargetFile); + + if (oldHash != newHash) + { + cause = $"hash for the file changed"; + return true; + } + + return false; + } + + public bool Save(string? cacheFilePath) + { + if (!Enabled || string.IsNullOrEmpty(cacheFilePath)) + return false; + + var json = JsonSerializer.Serialize (_newCache, new JsonSerializerOptions { WriteIndented = true }); + File.WriteAllText(cacheFilePath!, json); + return true; + } + + public ProxyFile NewFile(string targetFile) => new ProxyFile(targetFile, this); +} + +internal class ProxyFile +{ + public string TargetFile { get; } + public string TempFile { get; } + private FileCache _cache; + + public ProxyFile(string targetFile, FileCache cache) + { + _cache = cache; + this.TargetFile = targetFile; + this.TempFile = _cache.Enabled ? targetFile + ".tmp" : targetFile; + } + + public bool CopyOutputFileIfChanged() + { + if (!_cache.Enabled) + return true; + + if (!_cache.ShouldCopy(this, out string? cause)) + { + _cache.Log.LogMessage(MessageImportance.Low, $"Skipping copying over {TargetFile} as the contents are unchanged"); + return false; + } + + if (File.Exists(TargetFile)) + File.Delete(TargetFile); + + File.Copy(TempFile, TargetFile); + File.Delete(TempFile); + + _cache.Log.LogMessage(MessageImportance.Low, $"Copying {TempFile} to {TargetFile} because {cause}"); + return true; + } } public enum MonoAotMode @@ -737,3 +974,9 @@ public enum MonoAotModulesTableLanguage C, ObjC } + +internal class CompilerCache +{ + [JsonPropertyName("file_hashes")] + public ConcurrentDictionary FileHashes { get; set; } = new(); +} diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.csproj b/src/tasks/AotCompilerTask/MonoAOTCompiler.csproj index 338ba17ab82072..6371df56223568 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.csproj +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.csproj @@ -20,6 +20,7 @@ + diff --git a/src/tasks/Common/LogAsErrorException.cs b/src/tasks/Common/LogAsErrorException.cs new file mode 100644 index 00000000000000..a976de8cbbc4e2 --- /dev/null +++ b/src/tasks/Common/LogAsErrorException.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +public class LogAsErrorException : System.Exception +{ + public LogAsErrorException(string message) : base(message) + { + } +} diff --git a/src/tasks/Common/Utils.cs b/src/tasks/Common/Utils.cs index 1d04c4d27dcb0c..2c1a8b49370c7a 100644 --- a/src/tasks/Common/Utils.cs +++ b/src/tasks/Common/Utils.cs @@ -6,6 +6,7 @@ using System.Diagnostics; using System.IO; using System.Runtime.InteropServices; +using System.Security.Cryptography; using System.Text; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; @@ -205,6 +206,30 @@ internal static string CreateTemporaryBatchFile(string command) return file; } + public static bool CopyIfDifferent(string src, string dst, bool useHash) + { + if (!File.Exists(src)) + throw new ArgumentException($"Cannot find {src} file to copy", nameof(src)); + + bool areDifferent = !File.Exists(dst) || + (useHash && Utils.ComputeHash(src) != Utils.ComputeHash(dst)) || + (File.ReadAllText(src) != File.ReadAllText(dst)); + + if (areDifferent) + File.Copy(src, dst, true); + + return areDifferent; + } + + public static string ComputeHash(string filepath) + { + using var stream = File.OpenRead(filepath); + using HashAlgorithm hashAlgorithm = SHA512.Create(); + + byte[] hash = hashAlgorithm.ComputeHash(stream); + return Convert.ToBase64String(hash); + } + #if NETCOREAPP public static void DirectoryCopy(string sourceDir, string destDir, Func? predicate=null) { diff --git a/src/tasks/WasmAppBuilder/EmccCompile.cs b/src/tasks/WasmAppBuilder/EmccCompile.cs index eed8bb52690696..e869e98e8564ff 100644 --- a/src/tasks/WasmAppBuilder/EmccCompile.cs +++ b/src/tasks/WasmAppBuilder/EmccCompile.cs @@ -8,6 +8,7 @@ using System.IO; using System.Linq; using System.Text; +using System.Threading; using System.Threading.Tasks; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; @@ -39,6 +40,8 @@ public class EmccCompile : Microsoft.Build.Utilities.Task public ITaskItem[]? OutputFiles { get; private set; } private string? _tempPath; + private int _totalFiles; + private int _numCompiled; public override bool Execute() { @@ -61,10 +64,42 @@ public override bool Execute() return false; } + _totalFiles = SourceFiles.Length; IDictionary envVarsDict = GetEnvironmentVariablesDict(); ConcurrentBag outputItems = new(); try { + List<(string, string)> filesToCompile = new(); + foreach (ITaskItem srcItem in SourceFiles) + { + string srcFile = srcItem.ItemSpec; + string objFile = srcItem.GetMetadata("ObjectFile"); + string depMetadata = srcItem.GetMetadata("Dependencies"); + string[] depFiles = string.IsNullOrEmpty(depMetadata) + ? Array.Empty() + : depMetadata.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries); + + if (!ShouldCompile(srcFile, objFile, depFiles, out string reason)) + { + Log.LogMessage(MessageImportance.Low, $"Skipping {srcFile} because {reason}."); + } + else + { + Log.LogMessage(MessageImportance.Low, $"Compiling {srcFile} because {reason}."); + filesToCompile.Add((srcFile, objFile)); + } + } + + _numCompiled = SourceFiles.Length - filesToCompile.Count; + if (_numCompiled == _totalFiles) + { + // nothing to do! + return true; + } + + if (_numCompiled > 0) + Log.LogMessage(MessageImportance.High, $"[{_numCompiled}/{SourceFiles.Length}] skipped unchanged files"); + Log.LogMessage(MessageImportance.Low, "Using environment variables:"); foreach (var kvp in envVarsDict) Log.LogMessage(MessageImportance.Low, $"\t{kvp.Key} = {kvp.Value}"); @@ -76,32 +111,37 @@ public override bool Execute() Directory.CreateDirectory(_tempPath); int allowedParallelism = Math.Min(SourceFiles.Length, Environment.ProcessorCount); -#if false // Enable this when we bump msbuild to 16.1.0 if (BuildEngine is IBuildEngine9 be9) allowedParallelism = be9.RequestCores(allowedParallelism); -#endif if (DisableParallelCompile || allowedParallelism == 1) { - foreach (ITaskItem srcItem in SourceFiles) + foreach ((string srcFile, string outFile) in filesToCompile) { - if (!ProcessSourceFile(srcItem)) + if (!ProcessSourceFile(srcFile, outFile)) return false; } } else { - ParallelLoopResult result = Parallel.ForEach(SourceFiles, + ParallelLoopResult result = Parallel.ForEach(filesToCompile, new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism }, - (srcItem, state) => + (toCompile, state) => { - if (!ProcessSourceFile(srcItem)) + if (!ProcessSourceFile(toCompile.Item1, toCompile.Item2)) state.Stop(); }); if (!result.IsCompleted && !Log.HasLoggedErrors) Log.LogError("Unknown failed occured while compiling"); } + + if (!Log.HasLoggedErrors) + { + int numUnchanged = _totalFiles - _numCompiled; + if (numUnchanged > 0) + Log.LogMessage(MessageImportance.High, $"[{numUnchanged}/{_totalFiles}] unchanged."); + } } finally { @@ -112,14 +152,13 @@ public override bool Execute() OutputFiles = outputItems.ToArray(); return !Log.HasLoggedErrors; - bool ProcessSourceFile(ITaskItem srcItem) + bool ProcessSourceFile(string srcFile, string objFile) { - string srcFile = srcItem.ItemSpec; - string objFile = srcItem.GetMetadata("ObjectFile"); - + string tmpObjFile = Path.GetTempFileName(); try { - string command = $"emcc {Arguments} -c -o \"{objFile}\" \"{srcFile}\""; + string command = $"emcc {Arguments} -c -o \"{tmpObjFile}\" \"{srcFile}\""; + var startTime = DateTime.Now; // Log the command in a compact format which can be copy pasted StringBuilder envStr = new StringBuilder(string.Empty); @@ -135,16 +174,26 @@ bool ProcessSourceFile(ITaskItem srcItem) debugMessageImportance: messageImportance, label: Path.GetFileName(srcFile)); + var endTime = DateTime.Now; + var elapsedSecs = (endTime - startTime).TotalSeconds; if (exitCode != 0) { - Log.LogError($"Failed to compile {srcFile} -> {objFile}"); + Log.LogError($"Failed to compile {srcFile} -> {objFile}{Environment.NewLine}{output} [took {elapsedSecs:F}s]"); return false; } + if (!Utils.CopyIfDifferent(tmpObjFile, objFile, useHash: true)) + Log.LogMessage(MessageImportance.Low, $"Did not overwrite {objFile} as the contents are unchanged"); + else + Log.LogMessage(MessageImportance.Low, $"Copied {tmpObjFile} to {objFile}"); + ITaskItem newItem = new TaskItem(objFile); newItem.SetMetadata("SourceFile", srcFile); outputItems.Add(newItem); + int count = Interlocked.Increment(ref _numCompiled); + Log.LogMessage(MessageImportance.High, $"[{count}/{_totalFiles}] {Path.GetFileName(srcFile)} -> {Path.GetFileName(objFile)} [took {elapsedSecs:F}s]"); + return !Log.HasLoggedErrors; } catch (Exception ex) @@ -152,6 +201,58 @@ bool ProcessSourceFile(ITaskItem srcItem) Log.LogError($"Failed to compile {srcFile} -> {objFile}{Environment.NewLine}{ex.Message}"); return false; } + finally + { + File.Delete(tmpObjFile); + } + } + } + + private bool ShouldCompile(string srcFile, string objFile, string[] depFiles, out string reason) + { + if (!File.Exists(srcFile)) + throw new ArgumentException($"Could not find source file {srcFile}"); + + if (!File.Exists(objFile)) + { + reason = $"output file {objFile} doesn't exist"; + return true; + } + + if (IsNewerThanOutput(srcFile, objFile, out reason)) + return true; + + foreach (string depFile in depFiles) + { + if (IsNewerThanOutput(depFile, objFile, out reason)) + return true; + } + + reason = "everything is up-to-date."; + return false; + + bool IsNewerThanOutput(string inFile, string outFile, out string reason) + { + if (!File.Exists(inFile)) + { + reason = $"Could not find dependency file {inFile} needed for compiling {srcFile} to {outFile}"; + Log.LogWarning(reason); + return true; + } + + DateTime lastWriteTimeSrc = File.GetLastWriteTimeUtc(inFile); + DateTime lastWriteTimeDst = File.GetLastWriteTimeUtc(outFile); + + if (lastWriteTimeSrc > lastWriteTimeDst) + { + reason = $"{inFile} is newer than {outFile}"; + return true; + } + else + { + reason = $"{inFile} is older than {outFile}"; + return false; + } } } diff --git a/src/tasks/WasmAppBuilder/IcallTableGenerator.cs b/src/tasks/WasmAppBuilder/IcallTableGenerator.cs index e0737521dbfa30..13c75c39bc64cd 100644 --- a/src/tasks/WasmAppBuilder/IcallTableGenerator.cs +++ b/src/tasks/WasmAppBuilder/IcallTableGenerator.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using System.Text; @@ -19,7 +20,7 @@ public class IcallTableGenerator : Task public string? RuntimeIcallTableFile { get; set; } [Required] public ITaskItem[]? Assemblies { get; set; } - [Required] + [Required, NotNull] public string? OutputPath { get; set; } private List _icalls = new List (); @@ -27,7 +28,6 @@ public class IcallTableGenerator : Task public override bool Execute() { - Log.LogMessage(MessageImportance.Normal, $"Generating icall table to '{OutputPath}'."); GenIcallTable(RuntimeIcallTableFile!, Assemblies!.Select(item => item.ItemSpec).ToArray()); return true; } @@ -50,8 +50,16 @@ public void GenIcallTable(string runtimeIcallTableFile, string[] assemblies) ProcessType(type); } - using (var w = File.CreateText(OutputPath!)) + string tmpFileName = Path.GetTempFileName(); + using (var w = File.CreateText(tmpFileName)) EmitTable (w); + + if (Utils.CopyIfDifferent(tmpFileName, OutputPath, useHash: false)) + Log.LogMessage(MessageImportance.Low, $"Generating icall table to '{OutputPath}'."); + else + Log.LogMessage(MessageImportance.Low, $"Icall table in {OutputPath} is unchanged."); + + File.Delete(tmpFileName); } private void EmitTable (StreamWriter w) diff --git a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs index daf0b8afc56eed..f6b89533f98632 100644 --- a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs +++ b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using System.Text; @@ -19,14 +20,14 @@ public class PInvokeTableGenerator : Task public ITaskItem[]? Modules { get; set; } [Required] public ITaskItem[]? Assemblies { get; set; } - [Required] + + [Required, NotNull] public string? OutputPath { get; set; } private static char[] s_charsToReplace = new[] { '.', '-', }; public override bool Execute() { - Log.LogMessage(MessageImportance.Normal, $"Generating pinvoke table to '{OutputPath}'."); GenPInvokeTable(Modules!.Select(item => item.ItemSpec).ToArray(), Assemblies!.Select(item => item.ItemSpec).ToArray()); return true; } @@ -49,11 +50,19 @@ public void GenPInvokeTable(string[] pinvokeModules, string[] assemblies) CollectPInvokes(pinvokes, callbacks, type); } - using (var w = File.CreateText(OutputPath!)) + string tmpFileName = Path.GetTempFileName(); + using (var w = File.CreateText(tmpFileName)) { EmitPInvokeTable(w, modules, pinvokes); EmitNativeToInterp(w, callbacks); } + + if (Utils.CopyIfDifferent(tmpFileName, OutputPath, useHash: false)) + Log.LogMessage(MessageImportance.Low, $"Generating pinvoke table to '{OutputPath}'."); + else + Log.LogMessage(MessageImportance.Low, $"PInvoke table in {OutputPath} is unchanged."); + + File.Delete(tmpFileName); } private void CollectPInvokes(List pinvokes, List callbacks, Type type) @@ -119,7 +128,7 @@ private void EmitPInvokeTable(StreamWriter w, Dictionary modules Where(l => l.Module == module && !l.Skip). OrderBy(l => l.EntryPoint). GroupBy(d => d.EntryPoint). - Select (l => "{\"" + l.Key + "\", " + l.Key + "}, // " + string.Join (", ", l.Select(c => c.Method.DeclaringType!.Module!.Assembly!.GetName ()!.Name!).Distinct())); + Select (l => "{\"" + l.Key + "\", " + l.Key + "}, // " + string.Join (", ", l.Select(c => c.Method.DeclaringType!.Module!.Assembly!.GetName ()!.Name!).Distinct().OrderBy(n => n))); foreach (var pinvoke in assemblies_pinvokes) { w.WriteLine (pinvoke); diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs index 46c0174148be25..57cb7c383f107a 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs +++ b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs @@ -248,12 +248,14 @@ public override bool Execute () config.Extra[name] = valueObject; } - string monoConfigPath = Path.Combine(AppDir, "mono-config.json"); - using (var sw = File.CreateText(monoConfigPath)) + string tmpMonoConfigPath = Path.GetTempFileName(); + using (var sw = File.CreateText(tmpMonoConfigPath)) { var json = JsonSerializer.Serialize (config, new JsonSerializerOptions { WriteIndented = true }); sw.Write(json); } + string monoConfigPath = Path.Combine(AppDir, "mono-config.json"); + Utils.CopyIfDifferent(tmpMonoConfigPath, monoConfigPath, useHash: false); _fileWrites.Add(monoConfigPath); if (ExtraFilesToDeploy != null) diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildAndRunAttribute.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildAndRunAttribute.cs index 46e7215505f6ab..c49b19517b3f2a 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildAndRunAttribute.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildAndRunAttribute.cs @@ -20,21 +20,28 @@ namespace Wasm.Build.Tests [AttributeUsage(AttributeTargets.Method, AllowMultiple = true, Inherited = true)] public class BuildAndRunAttribute : DataAttribute { - private bool _aot; - private RunHost _host; - private object?[] _parameters; + private readonly IEnumerable _data; - public BuildAndRunAttribute(bool aot=false, RunHost host = RunHost.All, params object?[] parameters) + public BuildAndRunAttribute(BuildArgs buildArgs, RunHost host = RunHost.All, params object?[] parameters) { - this._aot = aot; - this._host = host; - this._parameters = parameters; + _data = new IEnumerable[] + { + new object?[] { buildArgs }.AsEnumerable(), + } + .AsEnumerable() + .Multiply(parameters) + .WithRunHosts(host) + .UnwrapItemsAsArrays().ToList().Dump(); } - public override IEnumerable GetData(MethodInfo testMethod) - => BuildTestBase.ConfigWithAOTData(_aot) - .Multiply(_parameters) - .WithRunHosts(_host) + public BuildAndRunAttribute(bool aot=false, RunHost host = RunHost.All, params object?[] parameters) + { + _data = BuildTestBase.ConfigWithAOTData(aot) + .Multiply(parameters) + .WithRunHosts(host) .UnwrapItemsAsArrays().ToList().Dump(); + } + + public override IEnumerable GetData(MethodInfo testMethod) => _data; } } diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildTestBase.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildTestBase.cs index e5d0b2e78cd707..3cd21151a07859 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildTestBase.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildTestBase.cs @@ -94,25 +94,29 @@ public BuildTestBase(ITestOutputHelper output, SharedBuildPerTestClassFixture bu - aot but no wrapper - check that AppBundle wasn't generated */ - public static IEnumerable> ConfigWithAOTData(bool aot) - => new IEnumerable[] + public static IEnumerable> ConfigWithAOTData(bool aot, string? config=null) + { + if (config == null) + { + return new IEnumerable[] + { + #if TEST_DEBUG_CONFIG_ALSO + // list of each member data - for Debug+@aot + new object?[] { new BuildArgs("placeholder", "Debug", aot, "placeholder", string.Empty) }.AsEnumerable(), + #endif + // list of each member data - for Release+@aot + new object?[] { new BuildArgs("placeholder", "Release", aot, "placeholder", string.Empty) }.AsEnumerable() + }.AsEnumerable(); + } + else + { + return new IEnumerable[] { -#if TEST_DEBUG_CONFIG_ALSO - // list of each member data - for Debug+@aot - new object?[] { new BuildArgs("placeholder", "Debug", aot, "placeholder", string.Empty) }.AsEnumerable(), -#endif - - // list of each member data - for Release+@aot - new object?[] { new BuildArgs("placeholder", "Release", aot, "placeholder", string.Empty) }.AsEnumerable() - }.AsEnumerable(); - - public static IEnumerable BuildAndRunData(bool aot = false, - RunHost host = RunHost.All, - params object[] parameters) - => ConfigWithAOTData(aot) - .Multiply(parameters) - .WithRunHosts(host) - .UnwrapItemsAsArrays(); + new object?[] { new BuildArgs("placeholder", config, aot, "placeholder", string.Empty) }.AsEnumerable() + }; + } + } + protected string RunAndTestWasmApp(BuildArgs buildArgs, RunHost host, @@ -279,7 +283,8 @@ protected static BuildArgs ExpandBuildArgs(BuildArgs buildArgs, string extraProp bool hasIcudt = true, bool useCache = true, bool expectSuccess = true, - bool createProject = true) + bool createProject = true, + string? verbosity=null) { if (useCache && _buildContext.TryGetBuildFor(buildArgs, out BuildProduct? product)) { @@ -318,7 +323,7 @@ protected static BuildArgs ExpandBuildArgs(BuildArgs buildArgs, string extraProp _testOutput.WriteLine($"Binlog path: {logFilePath}"); Console.WriteLine($"Binlog path: {logFilePath}"); sb.Append($" /bl:\"{logFilePath}\" /nologo"); - sb.Append($" /v:diag /fl /flp:\"v:diag,LogFile={logFilePath}.log\" /v:minimal"); + sb.Append($" /fl /flp:\"v:diag,LogFile={logFilePath}.log\" /v:{verbosity ?? "minimal"}"); if (buildArgs.ExtraBuildArgs != null) sb.Append($" {buildArgs.ExtraBuildArgs} "); @@ -426,8 +431,8 @@ protected static void AssertFilesExist(string dir, string[] filenames, string? l { Assert.True(File.Exists(path), label != null - ? $"{label}: {path} doesn't exist" - : $"{path} doesn't exist"); + ? $"{label}: File exists: {path}" + : $"File exists: {path}"); } else { @@ -451,9 +456,9 @@ protected static void AssertFile(string file0, string file1, string? label=null, FileInfo finfo1 = new(file1); if (same) - Assert.True(finfo0.Length == finfo1.Length, $"{label}: File sizes don't match for {file0} ({finfo0.Length}), and {file1} ({finfo1.Length})"); + Assert.True(finfo0.Length == finfo1.Length, $"{label}:{Environment.NewLine} File sizes don't match for {file0} ({finfo0.Length}), and {file1} ({finfo1.Length})"); else - Assert.True(finfo0.Length != finfo1.Length, $"{label}: File sizes should not match for {file0} ({finfo0.Length}), and {file1} ({finfo1.Length})"); + Assert.True(finfo0.Length != finfo1.Length, $"{label}:{Environment.NewLine} File sizes should not match for {file0} ({finfo0.Length}), and {file1} ({finfo1.Length})"); } protected (int exitCode, string buildOutput) AssertBuild(string args, string label="build", bool expectSuccess=true, IDictionary? envVars=null, int? timeoutMs=null) @@ -474,6 +479,13 @@ protected string GetBinDir(string config, string targetFramework=s_targetFramewo return Path.Combine(dir!, "bin", config, targetFramework, "browser-wasm"); } + protected string GetObjDir(string config, string targetFramework=s_targetFramework, string? baseDir=null) + { + var dir = baseDir ?? _projectDir; + Assert.NotNull(dir); + return Path.Combine(dir!, "obj", config, targetFramework, "browser-wasm"); + } + public static (int exitCode, string buildOutput) RunProcess(string path, ITestOutputHelper _testOutput, string args = "", @@ -591,11 +603,8 @@ void LogData(string label, string? message) public void Dispose() { - if (s_skipProjectCleanup || !_enablePerTestCleanup) - return; - - if (_projectDir != null) - _buildContext.RemoveFromCache(_projectDir); + if (_projectDir != null && _enablePerTestCleanup) + _buildContext.RemoveFromCache(_projectDir, keepDir: s_skipProjectCleanup); } private static string GetEnvironmentVariableOrDefault(string envVarName, string defaultValue) @@ -613,6 +622,10 @@ public static int Main() }"; } - public record BuildArgs(string ProjectName, string Config, bool AOT, string ProjectFileContents, string? ExtraBuildArgs); + public record BuildArgs(string ProjectName, + string Config, + bool AOT, + string ProjectFileContents, + string? ExtraBuildArgs); public record BuildProduct(string ProjectDir, string LogFile, bool Result); } diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/HelperExtensions.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/HelperExtensions.cs index e3d1ca3dd37b2a..a70a51bb10bbd8 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/HelperExtensions.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/HelperExtensions.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Linq; using System.IO; +using System.Text; #nullable enable @@ -92,5 +93,23 @@ public static class HelperExtensions .Append((object?)runId)); }); } + + public static void UpdateTo(this IDictionary dict, bool unchanged, params string[] filenames) + { + foreach (var filename in filenames) + { + if (!dict.TryGetValue(filename, out var oldValue)) + { + StringBuilder sb = new(); + sb.AppendLine($"Cannot find key named {filename} in the dict. Existing ones:"); + foreach (var kvp in dict) + sb.AppendLine($"[{kvp.Key}] = [{kvp.Value}]"); + + throw new KeyNotFoundException(sb.ToString()); + } + + dict[filename] = (oldValue.fullPath, unchanged); + } + } } } diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/FlagsChangeRebuildTest.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/FlagsChangeRebuildTest.cs new file mode 100644 index 00000000000000..7ebc2d05cdd955 --- /dev/null +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/FlagsChangeRebuildTest.cs @@ -0,0 +1,118 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Collections.Generic; +using System.Linq; +using Xunit; +using Xunit.Abstractions; + +#nullable enable + +namespace Wasm.Build.Tests +{ + public class FlagsChangeRebuildTest : NativeRebuildTestsBase + { + public FlagsChangeRebuildTest(ITestOutputHelper output, SharedBuildPerTestClassFixture buildContext) + : base(output, buildContext) + { + } + + public static IEnumerable FlagsChangesForNativeRelinkingData(bool aot) + => ConfigWithAOTData(aot, config: "Release").Multiply( + new object[] { /*cflags*/ "/p:EmccExtraCFlags=-g", /*ldflags*/ "" }, + new object[] { /*cflags*/ "", /*ldflags*/ "/p:EmccExtraLDFlags=-g" }, + new object[] { /*cflags*/ "/p:EmccExtraCFlags=-g", /*ldflags*/ "/p:EmccExtraLDFlags=-g" } + ).WithRunHosts(RunHost.V8).UnwrapItemsAsArrays().Dump(); + + [Theory] + [MemberData(nameof(FlagsChangesForNativeRelinkingData), parameters: /*aot*/ false)] + [MemberData(nameof(FlagsChangesForNativeRelinkingData), parameters: /*aot*/ true)] + public void ExtraEmccFlagsSetButNoRealChange(BuildArgs buildArgs, string extraCFlags, string extraLDFlags, RunHost host, string id) + { + buildArgs = buildArgs with { ProjectName = $"rebuild_flags_{buildArgs.Config}" }; + (buildArgs, BuildPaths paths) = FirstNativeBuild(s_mainReturns42, nativeRelink: true, invariant: false, buildArgs, id); + var pathsDict = GetFilesTable(buildArgs, paths, unchanged: true); + if (extraLDFlags.Length > 0) + pathsDict.UpdateTo(unchanged: false, "dotnet.wasm", "dotnet.js"); + + var originalStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + // Rebuild + + string mainAssembly = $"{buildArgs.ProjectName}.dll"; + string extraBuildArgs = $" {extraCFlags} {extraLDFlags}"; + string output = Rebuild(nativeRelink: true, invariant: false, buildArgs, id, extraBuildArgs: extraBuildArgs, verbosity: "normal"); + + var newStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + CompareStat(originalStat, newStat, pathsDict.Values); + + // cflags: pinvoke get's compiled, but doesn't overwrite pinvoke.o + // and thus doesn't cause relinking + AssertSubstring("pinvoke.c -> pinvoke.o", output, contains: extraCFlags.Length > 0); + + // ldflags: link step args change, so it should trigger relink + AssertSubstring("wasm-opt", output, contains: extraLDFlags.Length > 0); + + if (buildArgs.AOT) + { + // ExtraEmccLDFlags does not affect .bc files + Assert.DoesNotContain("Compiling assembly bitcode files", output); + } + + string runOutput = RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); + AssertSubstring($"Found statically linked AOT module '{Path.GetFileNameWithoutExtension(mainAssembly)}'", runOutput, + contains: buildArgs.AOT); + } + + public static IEnumerable FlagsOnlyChangeData(bool aot) + => ConfigWithAOTData(aot, config: "Release").Multiply( + new object[] { /*cflags*/ "/p:EmccCompileOptimizationFlag=-O1", /*ldflags*/ "" }, + new object[] { /*cflags*/ "", /*ldflags*/ "/p:EmccLinkOptimizationFlag=-O0" } + ).WithRunHosts(RunHost.V8).UnwrapItemsAsArrays().Dump(); + + [Theory] + [MemberData(nameof(FlagsOnlyChangeData), parameters: /*aot*/ false)] + [MemberData(nameof(FlagsOnlyChangeData), parameters: /*aot*/ true)] + public void OptimizationFlagChange(BuildArgs buildArgs, string cflags, string ldflags, RunHost host, string id) + { + // force _WasmDevel=false, so we don't get -O0 + buildArgs = buildArgs with { ProjectName = $"rebuild_flags_{buildArgs.Config}", ExtraBuildArgs = "/p:_WasmDevel=false" }; + (buildArgs, BuildPaths paths) = FirstNativeBuild(s_mainReturns42, nativeRelink: true, invariant: false, buildArgs, id); + + string mainAssembly = $"{buildArgs.ProjectName}.dll"; + var pathsDict = GetFilesTable(buildArgs, paths, unchanged: false); + pathsDict.UpdateTo(unchanged: true, mainAssembly, "icall-table.h", "pinvoke-table.h", "driver-gen.c"); + if (cflags.Length == 0) + pathsDict.UpdateTo(unchanged: true, "pinvoke.o", "corebindings.o", "driver.o"); + + pathsDict.Remove(mainAssembly); + if (buildArgs.AOT) + { + // link optimization flag change affects .bc->.o files too, but + // it might result in only *some* files being *changed, + // so, don't check for those + // Link optimization flag is set to Compile optimization flag, if unset + // so, it affects .bc files too! + foreach (string key in pathsDict.Keys.ToArray()) + { + if (key.EndsWith(".dll.bc", StringComparison.Ordinal) || key.EndsWith(".dll.o", StringComparison.Ordinal)) + pathsDict.Remove(key); + } + } + + var originalStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + // Rebuild + + string output = Rebuild(nativeRelink: true, invariant: false, buildArgs, id, extraBuildArgs: $" {cflags} {ldflags}", verbosity: "normal"); + var newStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + CompareStat(originalStat, newStat, pathsDict.Values); + + string runOutput = RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); + AssertSubstring($"Found statically linked AOT module '{Path.GetFileNameWithoutExtension(mainAssembly)}'", runOutput, + contains: buildArgs.AOT); + } + } +} diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NativeRebuildTestsBase.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NativeRebuildTestsBase.cs new file mode 100644 index 00000000000000..2256fa58386abc --- /dev/null +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NativeRebuildTestsBase.cs @@ -0,0 +1,209 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Collections.Generic; +using System.Linq; +using Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; +using System.Text; + +#nullable enable + +namespace Wasm.Build.Tests +{ + // TODO: test for runtime components + public class NativeRebuildTestsBase : BuildTestBase + { + public NativeRebuildTestsBase(ITestOutputHelper output, SharedBuildPerTestClassFixture buildContext) + : base(output, buildContext) + { + _enablePerTestCleanup = true; + } + + public static IEnumerable NativeBuildData() + { + List data = new(); + // relinking + data.AddRange(GetData(aot: false, nativeRelinking: true, invariant: false)); + data.AddRange(GetData(aot: false, nativeRelinking: true, invariant: true)); + + // aot + data.AddRange(GetData(aot: true, nativeRelinking: false, invariant: false)); + data.AddRange(GetData(aot: true, nativeRelinking: false, invariant: true)); + + return data; + + IEnumerable GetData(bool aot, bool nativeRelinking, bool invariant) + => ConfigWithAOTData(aot) + .Multiply(new object[] { nativeRelinking, invariant }) + .WithRunHosts(RunHost.V8) + .UnwrapItemsAsArrays().ToList().Dump(); + } + + internal (BuildArgs BuildArgs, BuildPaths paths) FirstNativeBuild(string programText, bool nativeRelink, bool invariant, BuildArgs buildArgs, string id, string extraProperties="") + { + buildArgs = GenerateProjectContents(buildArgs, nativeRelink, invariant, extraProperties); + BuildProject(buildArgs, + initProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText), + dotnetWasmFromRuntimePack: false, + hasIcudt: !invariant, + id: id, + createProject: true); + + RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: RunHost.V8, id: id); + return (buildArgs, GetBuildPaths(buildArgs)); + } + + protected string Rebuild(bool nativeRelink, bool invariant, BuildArgs buildArgs, string id, string extraProperties="", string extraBuildArgs="", string? verbosity=null) + { + if (!_buildContext.TryGetBuildFor(buildArgs, out BuildProduct? product)) + throw new XunitException($"Test bug: could not get the build product in the cache"); + + File.Move(product!.LogFile, Path.ChangeExtension(product.LogFile!, ".first.binlog")); + + buildArgs = buildArgs with { ExtraBuildArgs = $"{buildArgs.ExtraBuildArgs} {extraBuildArgs}" }; + var newBuildArgs = GenerateProjectContents(buildArgs, nativeRelink, invariant, extraProperties); + + // key(buildArgs) being changed + _buildContext.RemoveFromCache(product.ProjectDir); + _buildContext.CacheBuild(newBuildArgs, product); + + if (buildArgs.ProjectFileContents != newBuildArgs.ProjectFileContents) + File.WriteAllText(Path.Combine(_projectDir!, $"{buildArgs.ProjectName}.csproj"), buildArgs.ProjectFileContents); + buildArgs = newBuildArgs; + + _testOutput.WriteLine($"{Environment.NewLine}Rebuilding with no changes ..{Environment.NewLine}"); + (_, string output) = BuildProject(buildArgs, + id: id, + dotnetWasmFromRuntimePack: false, + hasIcudt: !invariant, + createProject: false, + useCache: false, + verbosity: verbosity); + + return output; + } + + protected BuildArgs GenerateProjectContents(BuildArgs buildArgs, bool nativeRelink, bool invariant, string extraProperties) + { + StringBuilder propertiesBuilder = new(); + propertiesBuilder.Append("<_WasmDevel>true"); + if (nativeRelink) + propertiesBuilder.Append($"true"); + if (invariant) + propertiesBuilder.Append($"true"); + propertiesBuilder.Append(extraProperties); + + return ExpandBuildArgs(buildArgs, propertiesBuilder.ToString()); + } + + internal void CompareStat(IDictionary oldStat, IDictionary newStat, IEnumerable<(string fullpath, bool unchanged)> expected) + { + StringBuilder msg = new(); + foreach (var expect in expected) + { + string expectFilename = Path.GetFileName(expect.fullpath); + if (!oldStat.TryGetValue(expectFilename, out FileStat? oldFs)) + { + msg.AppendLine($"Could not find an entry for {expectFilename} in old files"); + continue; + } + + if (!newStat.TryGetValue(expectFilename, out FileStat? newFs)) + { + msg.AppendLine($"Could not find an entry for {expectFilename} in new files"); + continue; + } + + bool actualUnchanged = oldFs == newFs; + if (expect.unchanged && !actualUnchanged) + { + msg.AppendLine($"[Expected unchanged file: {expectFilename}]{Environment.NewLine}" + + $" old: {oldFs}{Environment.NewLine}" + + $" new: {newFs}"); + } + else if (!expect.unchanged && actualUnchanged) + { + msg.AppendLine($"[Expected changed file: {expectFilename}]{Environment.NewLine}" + + $" {newFs}"); + } + } + + if (msg.Length > 0) + throw new XunitException($"CompareStat failed:{Environment.NewLine}{msg}"); + } + + internal IDictionary StatFiles(IEnumerable fullpaths) + { + Dictionary table = new(); + foreach (string file in fullpaths) + { + if (File.Exists(file)) + table.Add(Path.GetFileName(file), new FileStat(FullPath: file, Exists: true, LastWriteTimeUtc: File.GetLastWriteTimeUtc(file), Length: new FileInfo(file).Length)); + else + table.Add(Path.GetFileName(file), new FileStat(FullPath: file, Exists: false, LastWriteTimeUtc: DateTime.MinValue, Length: 0)); + } + + return table; + } + + internal BuildPaths GetBuildPaths(BuildArgs buildArgs) + { + string objDir = GetObjDir(buildArgs.Config); + string bundleDir = Path.Combine(GetBinDir(baseDir: _projectDir, config: buildArgs.Config), "AppBundle"); + string wasmDir = Path.Combine(objDir, "wasm"); + + return new BuildPaths(wasmDir, objDir, GetBinDir(buildArgs.Config), bundleDir); + } + + internal IDictionary GetFilesTable(BuildArgs buildArgs, BuildPaths paths, bool unchanged) + { + List files = new() + { + Path.Combine(paths.BinDir, "publish", $"{buildArgs.ProjectName}.dll"), + Path.Combine(paths.ObjWasmDir, "driver.o"), + Path.Combine(paths.ObjWasmDir, "corebindings.o"), + Path.Combine(paths.ObjWasmDir, "pinvoke.o"), + + Path.Combine(paths.ObjWasmDir, "icall-table.h"), + Path.Combine(paths.ObjWasmDir, "pinvoke-table.h"), + Path.Combine(paths.ObjWasmDir, "driver-gen.c"), + + Path.Combine(paths.BundleDir, "dotnet.wasm"), + Path.Combine(paths.BundleDir, "dotnet.js") + }; + + if (buildArgs.AOT) + { + files.AddRange(new[] + { + Path.Combine(paths.ObjWasmDir, $"{buildArgs.ProjectName}.dll.bc"), + Path.Combine(paths.ObjWasmDir, $"{buildArgs.ProjectName}.dll.o"), + + Path.Combine(paths.ObjWasmDir, "System.Private.CoreLib.dll.bc"), + Path.Combine(paths.ObjWasmDir, "System.Private.CoreLib.dll.o"), + }); + } + + var dict = new Dictionary(); + foreach (var file in files) + dict[Path.GetFileName(file)] = (file, unchanged); + + return dict; + } + + protected void AssertSubstring(string substring, string full, bool contains) + { + if (contains) + Assert.Contains(substring, full); + else + Assert.DoesNotContain(substring, full); + } + } + + internal record FileStat (bool Exists, DateTime LastWriteTimeUtc, long Length, string FullPath); + internal record BuildPaths(string ObjWasmDir, string ObjDir, string BinDir, string BundleDir); +} diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NoopNativeRebuildTest.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NoopNativeRebuildTest.cs new file mode 100644 index 00000000000000..1ddbd4467b6206 --- /dev/null +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NoopNativeRebuildTest.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Linq; +using Xunit; +using Xunit.Abstractions; + +#nullable enable + +namespace Wasm.Build.Tests +{ + public class NoopNativeRebuildTest : NativeRebuildTestsBase + { + public NoopNativeRebuildTest(ITestOutputHelper output, SharedBuildPerTestClassFixture buildContext) + : base(output, buildContext) + { + } + + [Theory] + [MemberData(nameof(NativeBuildData))] + public void NoOpRebuildForNativeBuilds(BuildArgs buildArgs, bool nativeRelink, bool invariant, RunHost host, string id) + { + buildArgs = buildArgs with { ProjectName = $"rebuild_noop_{buildArgs.Config}" }; + (buildArgs, BuildPaths paths) = FirstNativeBuild(s_mainReturns42, nativeRelink: nativeRelink, invariant: invariant, buildArgs, id); + + var pathsDict = GetFilesTable(buildArgs, paths, unchanged: true); + var originalStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + Rebuild(nativeRelink, invariant, buildArgs, id); + var newStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + CompareStat(originalStat, newStat, pathsDict.Values); + RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); + } + } +} diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/ReferenceNewAssemblyRebuildTest.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/ReferenceNewAssemblyRebuildTest.cs new file mode 100644 index 00000000000000..d908e287cf8bd1 --- /dev/null +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/ReferenceNewAssemblyRebuildTest.cs @@ -0,0 +1,57 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.IO; +using System.Linq; +using Xunit; +using Xunit.Abstractions; + +#nullable enable + +namespace Wasm.Build.Tests +{ + public class ReferenceNewAssemblyRebuildTest : NativeRebuildTestsBase + { + public ReferenceNewAssemblyRebuildTest(ITestOutputHelper output, SharedBuildPerTestClassFixture buildContext) + : base(output, buildContext) + { + } + + [Theory] + [MemberData(nameof(NativeBuildData))] + public void ReferenceNewAssembly(BuildArgs buildArgs, bool nativeRelink, bool invariant, RunHost host, string id) + { + buildArgs = buildArgs with { ProjectName = $"rebuild_tasks_{buildArgs.Config}" }; + (buildArgs, BuildPaths paths) = FirstNativeBuild(s_mainReturns42, nativeRelink, invariant: invariant, buildArgs, id); + + var pathsDict = GetFilesTable(buildArgs, paths, unchanged: false); + pathsDict.UpdateTo(unchanged: true, "corebindings.o"); + if (!buildArgs.AOT) // relinking + pathsDict.UpdateTo(unchanged: true, "driver-gen.c"); + + var originalStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + string programText = + @$" + using System; + using System.Text.Json; + public class Test + {{ + public static int Main() + {{" + + @" string json = ""{ \""name\"": \""value\"" }"";" + + @" var jdoc = JsonDocument.Parse($""{json}"", new JsonDocumentOptions());" + + @$" Console.WriteLine($""json: {{jdoc}}""); + return 42; + }} + }}"; + File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText); + + Rebuild(nativeRelink, invariant, buildArgs, id); + var newStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + CompareStat(originalStat, newStat, pathsDict.Values); + RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); + } + } +} diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/SimpleSourceChangeRebuildTest.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/SimpleSourceChangeRebuildTest.cs new file mode 100644 index 00000000000000..7f51447cb9e62a --- /dev/null +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/SimpleSourceChangeRebuildTest.cs @@ -0,0 +1,55 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.IO; +using System.Linq; +using Xunit; +using Xunit.Abstractions; + +#nullable enable + +namespace Wasm.Build.Tests +{ + public class SimpleSourceChangeRebuildTest : NativeRebuildTestsBase + { + public SimpleSourceChangeRebuildTest(ITestOutputHelper output, SharedBuildPerTestClassFixture buildContext) + : base(output, buildContext) + { + } + + [Theory] + [MemberData(nameof(NativeBuildData))] + public void SimpleStringChangeInSource(BuildArgs buildArgs, bool nativeRelink, bool invariant, RunHost host, string id) + { + buildArgs = buildArgs with { ProjectName = $"rebuild_simple_{buildArgs.Config}" }; + (buildArgs, BuildPaths paths) = FirstNativeBuild(s_mainReturns42, nativeRelink, invariant: invariant, buildArgs, id); + + string mainAssembly = $"{buildArgs.ProjectName}.dll"; + var pathsDict = GetFilesTable(buildArgs, paths, unchanged: true); + pathsDict.UpdateTo(unchanged: false, mainAssembly); + pathsDict.UpdateTo(unchanged: !buildArgs.AOT, "dotnet.wasm", "dotnet.js"); + + if (buildArgs.AOT) + pathsDict.UpdateTo(unchanged: false, $"{mainAssembly}.bc", $"{mainAssembly}.o"); + + var originalStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + // Changes + string mainResults55 = @" + public class TestClass { + public static int Main() + { + return 55; + } + }"; + File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), mainResults55); + + // Rebuild + Rebuild(nativeRelink, invariant, buildArgs, id); + var newStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + CompareStat(originalStat, newStat, pathsDict.Values); + RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 55, host: host, id: id); + } + } +} diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/RebuildTests.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/RebuildTests.cs index 769d22fa660dc9..961c141348275d 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/RebuildTests.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/RebuildTests.cs @@ -2,7 +2,9 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Collections.Generic; using System.IO; +using System.Linq; using Xunit; using Xunit.Abstractions; using Xunit.Sdk; @@ -18,21 +20,23 @@ public RebuildTests(ITestOutputHelper output, SharedBuildPerTestClassFixture bui { } + public static IEnumerable NonNativeDebugRebuildData() + => ConfigWithAOTData(aot: false, config: "Debug") + .WithRunHosts(RunHost.V8) + .UnwrapItemsAsArrays().ToList(); + [Theory] - [BuildAndRun(host: RunHost.V8, aot: false, parameters: false)] - [BuildAndRun(host: RunHost.V8, aot: false, parameters: true)] - [BuildAndRun(host: RunHost.V8, aot: true, parameters: false)] - public void NoOpRebuild(BuildArgs buildArgs, bool nativeRelink, RunHost host, string id) + [MemberData(nameof(NonNativeDebugRebuildData))] + public void NoOpRebuild(BuildArgs buildArgs, RunHost host, string id) { string projectName = $"rebuild_{buildArgs.Config}_{buildArgs.AOT}"; - bool dotnetWasmFromRuntimePack = !nativeRelink && !buildArgs.AOT; buildArgs = buildArgs with { ProjectName = projectName }; - buildArgs = ExpandBuildArgs(buildArgs, $"{(nativeRelink ? "true" : "false")}"); + buildArgs = ExpandBuildArgs(buildArgs); BuildProject(buildArgs, initProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), - dotnetWasmFromRuntimePack: dotnetWasmFromRuntimePack, + dotnetWasmFromRuntimePack: true, id: id, createProject: true); @@ -48,7 +52,7 @@ public void NoOpRebuild(BuildArgs buildArgs, bool nativeRelink, RunHost host, st // no-op Rebuild BuildProject(buildArgs, id: id, - dotnetWasmFromRuntimePack: dotnetWasmFromRuntimePack, + dotnetWasmFromRuntimePack: true, createProject: false, useCache: false); diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/SharedBuildPerTestClassFixture.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/SharedBuildPerTestClassFixture.cs index e84a151bd5b02d..03437e67302133 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/SharedBuildPerTestClassFixture.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/SharedBuildPerTestClassFixture.cs @@ -18,14 +18,15 @@ public class SharedBuildPerTestClassFixture : IDisposable public void CacheBuild(BuildArgs buildArgs, BuildProduct product) => _buildPaths.Add(buildArgs, product); - public void RemoveFromCache(string buildPath) + public void RemoveFromCache(string buildPath, bool keepDir=true) { KeyValuePair? foundKvp = _buildPaths.Where(kvp => kvp.Value.ProjectDir == buildPath).SingleOrDefault(); if (foundKvp == null) throw new Exception($"Could not find build path {buildPath} in cache to remove."); _buildPaths.Remove(foundKvp.Value.Key); - RemoveDirectory(buildPath); + if (!keepDir) + RemoveDirectory(buildPath); } public bool TryGetBuildFor(BuildArgs buildArgs, [NotNullWhen(true)] out BuildProduct? product) @@ -42,6 +43,9 @@ public void Dispose() private void RemoveDirectory(string path) { + if (EnvironmentVariables.SkipProjectCleanup == "1") + return; + try { Directory.Delete(path, recursive: true); diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/Wasm.Build.Tests.csproj b/src/tests/BuildWasmApps/Wasm.Build.Tests/Wasm.Build.Tests.csproj index 2994b3f7acbaac..7dacbc3ec26f7e 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/Wasm.Build.Tests.csproj +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/Wasm.Build.Tests.csproj @@ -62,6 +62,10 @@ dotnet exec xunit.console.dll $(AssemblyName).dll -xml %24XHARNESS_OUT/testResults.xml dotnet.exe exec xunit.console.dll $(AssemblyName).dll -xml %XHARNESS_OUT%\testResults.xml + + $(RunScriptCommand) %24HELIX_XUNIT_ARGS + $(RunScriptCommand) %HELIX_XUNIT_ARGS% + $(RunScriptCommand) -nocolor $(RunScriptCommand) -verbose