diff --git a/docs/README.md b/docs/README.md index 023cb7b5686099..6989879155dcc5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -25,6 +25,7 @@ Design Docs ================= - [.NET Globalization Invariant Mode](design/features/globalization-invariant-mode.md) +- [WASM Globalization Icu](design/features/globalization-icu-wasm.md) - [Cross-Platform Cryptography](design/features/cross-platform-cryptography.md) - Many more under [design/features](design/features/) diff --git a/docs/design/features/globalization-icu-wasm.md b/docs/design/features/globalization-icu-wasm.md new file mode 100644 index 00000000000000..dc83b154feff44 --- /dev/null +++ b/docs/design/features/globalization-icu-wasm.md @@ -0,0 +1,63 @@ +# WASM Globalization Icu + +In WASM applications when [globalization invariant mode](globalization-invariant-mode.md) is switched off, internalization data file is loaded. There are four basic types of these files: +- `icudt.dat` - full data +- `icudt_EFIGS.dat` - data for locales: "en-*", "fr-FR", "es-ES", "it-IT", and "de-DE". +- `icudt_CJK.dat` - for locales: "en" "ja", "ko", and "zh". +- `icudt_no_CJK.dat` - all locales from `icudt.dat`, excluding "ja", "ko", and "zh". + +## Wasm Console, Wasm Browser + +We can specify the file we want to load, e.g. `icudt_no_CJK.dat` by adding to .csproj: +``` +icudt_no_CJK.dat +``` +Only one value for `WasmIcuDataFileName` can be set. It can also be a custom file, created by the developer. To create a custom ICU file, see `Custom ICU` section below. If no `WasmIcuDataFileName` was specified, the application's culture will be checked and the corresponding file will be loaded, e.g. for `en-US` file `icudt_EFIGS.dat`, and for `zh-CN` - `icudt_CJK.dat`. + +## Custom ICU + +The easiest way to build ICU is to open https://github.com/dotnet/icu/ it in [Codespaces](docs\workflow\Codespaces.md). See files in https://github.com/dotnet/icu/tree/dotnet/main/icu-filters, and read https://unicode-org.github.io/icu/userguide/icu_data/buildtool.html#locale-slicing. Build your own filter or edit the existing file. +We advise to edit the filters **only by adding/removing locales** from the `localeFilter/whitelist` to avoid removing important data. We recommend not to remove "en-US" locale from the localeFilter/whitelist because it is used as a fallback. Removing it for when +- `true`: results in `Encountered infinite recursion while looking for resource in System.Private.Corelib.` exception +- when predefined cultures only is not set: results in resolving data from ICU's `root.txt` files, e.g. `CultureInfo.DateTimeFormat.GetDayName(DateTime.Today.DayOfWeek)` will return an abbreviated form: `Mon` instead of `Monday`. +Removing specific feature data might result in an exception that starts with `[CultureData.IcuGetLocaleInfo(LocaleStringData)] Failed`. It means you removed data necessary to extract basic information about the locale. + + In the file `eng/icu.mk`, you can choose what filters to build. Choose the platform: + +### Building for Browser: +* For prerequisites run `.devcontainer/postCreateCommand.sh` (it is run automatically on creation if using Codespaces) +* Building: + ``` + ./build.sh /p:TargetOS=Browser /p:TargetArchitecture=wasm /p:IcuTracing=true + ``` + Output is located in `artifacts/bin/icu-browser-wasm`. + +### Building for Mobiles: +* For prerequisites run: + ```bash + export ANDROID_NDK_ROOT=$PWD/artifacts/ndk/ + mkdir $ANDROID_NDK_ROOT + wget https://dl.google.com/android/repository/android-ndk-r25b-linux.zip + unzip android-ndk-r25b-linux.zip -d $ANDROID_NDK_ROOT + rm android-ndk-r25b-linux.zip + mv $ANDROID_NDK_ROOT/*/* $ANDROID_NDK_ROOT + rmdir $ANDROID_NDK_ROOT/android-ndk-r25b + ``` +* Building: + ```bash + ./build.sh /p:TargetOS=Android /p:TargetArchitecture=x64 /p:IcuTracing=true + ``` + +Output from both builds will be located in subdirectories of `artifacts/bin`. Copy the generated `.dat` files to a suitable location and provide the full path to it in the `.csproj`, e.g.: +```xml +C:\Users\wasmUser\icuSources\customIcu.dat +``` + +## Blazor + +In Blazor we are loading the file based on the applications's culture. +To force the full data to be loaded, add this to your `.csproj`: +```xml +true +``` +Custom files loading for Blazor is not possible. diff --git a/eng/testing/scenarios/BuildWasmAppsJobsList.txt b/eng/testing/scenarios/BuildWasmAppsJobsList.txt index 6bafc00c1ad435..38e288f81cfa14 100644 --- a/eng/testing/scenarios/BuildWasmAppsJobsList.txt +++ b/eng/testing/scenarios/BuildWasmAppsJobsList.txt @@ -4,16 +4,18 @@ Wasm.Build.NativeRebuild.Tests.OptimizationFlagChangeTests Wasm.Build.NativeRebuild.Tests.ReferenceNewAssemblyRebuildTest Wasm.Build.NativeRebuild.Tests.SimpleSourceChangeRebuildTest Wasm.Build.Tests.Blazor.BuildPublishTests -Wasm.Build.Tests.Blazor.MiscTests2 Wasm.Build.Tests.Blazor.MiscTests +Wasm.Build.Tests.Blazor.MiscTests2 Wasm.Build.Tests.Blazor.NativeRefTests Wasm.Build.Tests.BuildPublishTests Wasm.Build.Tests.CleanTests Wasm.Build.Tests.ConfigSrcTests +Wasm.Build.Tests.IcuShardingTests Wasm.Build.Tests.InvariantGlobalizationTests Wasm.Build.Tests.MainWithArgsTests Wasm.Build.Tests.NativeBuildTests Wasm.Build.Tests.NativeLibraryTests +Wasm.Build.Tests.NonWasmTemplateBuildTests Wasm.Build.Tests.PInvokeTableGeneratorTests Wasm.Build.Tests.RebuildTests Wasm.Build.Tests.SatelliteAssembliesTests diff --git a/eng/testing/tests.browser.targets b/eng/testing/tests.browser.targets index 6f0972fbfa110e..849a12e96a83b4 100644 --- a/eng/testing/tests.browser.targets +++ b/eng/testing/tests.browser.targets @@ -8,6 +8,9 @@ true + + true + $([MSBuild]::NormalizeDirectory($(MonoProjectRoot), 'wasm', 'emsdk')) @@ -184,6 +187,8 @@ <_WasmPropertyNames Include="DisableParallelEmccCompile" /> <_WasmPropertyNames Include="EmccCompileOptimizationFlag" /> <_WasmPropertyNames Include="EmccLinkOptimizationFlag" /> + <_WasmPropertyNames Include="WasmIncludeFullIcuData" /> + <_WasmPropertyNames Include="WasmIcuDataFileName" /> diff --git a/src/mono/wasi/build/WasiApp.targets b/src/mono/wasi/build/WasiApp.targets index 0688e932d0e164..56a05ab2506778 100644 --- a/src/mono/wasi/build/WasiApp.targets +++ b/src/mono/wasi/build/WasiApp.targets @@ -319,27 +319,29 @@ - icudt.dat - <_HasDotnetWasm Condition="'%(WasmNativeAsset.FileName)%(WasmNativeAsset.Extension)' == 'dotnet.wasm'">true - - - + + + + + + DependsOnTargets="_WasmGenerateRuntimeConfig;_GetWasiGenerateAppBundleDependencies;_WasiDefaultGenerateAppBundle;_GenerateRunWasmtimeScript"> + + @@ -353,7 +355,7 @@ DefaultHostConfig="$(DefaultWasmHostConfig)" InvariantGlobalization="$(InvariantGlobalization)" SatelliteAssemblies="@(_WasmSatelliteAssemblies)" - IcuDataFileName="$(WasmIcuDataFileName)" + IcuDataFileNames="@(WasmIcuDataFileNames)" ExtraFilesToDeploy="@(WasmExtraFilesToDeploy)" NativeAssets="@(WasmNativeAsset)" DebugLevel="$(WasmDebugLevel)" diff --git a/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs b/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs index 714ad4075bb4d5..39e02f4c799690 100644 --- a/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs +++ b/src/mono/wasm/Wasm.Build.Tests/BuildTestBase.cs @@ -30,6 +30,7 @@ public abstract class BuildTestBase : IClassFixture skip automatic icu testing with Node + // on Linux sharding does not work because we rely on LANG env var to check locale and emcc is overwriting it + protected static RunHost s_hostsForOSLocaleSensitiveTests = RunHost.Chrome; // FIXME: use an envvar to override this - protected static int s_defaultPerTestTimeoutMs = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? 30*60*1000 : 15*60*1000; + protected static int s_defaultPerTestTimeoutMs = s_isWindows ? 30*60*1000 : 15*60*1000; protected static BuildEnvironment s_buildEnv; private const string s_runtimePackPathPattern = "\\*\\* MicrosoftNetCoreAppRuntimePackDir : '([^ ']*)'"; private const string s_nugetInsertionTag = ""; @@ -139,7 +145,8 @@ protected string RunAndTestWasmApp(BuildArgs buildArgs, string targetFramework = DefaultTargetFramework, string? extraXHarnessMonoArgs = null, string? extraXHarnessArgs = null, - string jsRelativePath = "test-main.js") + string jsRelativePath = "test-main.js", + string environmentLocale = DefaultEnvironmentLocale) { buildDir ??= _projectDir; envVars ??= new(); @@ -157,16 +164,15 @@ protected string RunAndTestWasmApp(BuildArgs buildArgs, } bundleDir ??= Path.Combine(GetBinDir(baseDir: buildDir, config: buildArgs.Config, targetFramework: targetFramework), "AppBundle"); - if (host is RunHost.V8 && OperatingSystem.IsWindows()) + IHostRunner hostRunner = GetHostRunnerFromRunHost(host); + if (!hostRunner.CanRunWBT()) throw new InvalidOperationException("Running tests with V8 on windows isn't supported"); // Use wasm-console.log to get the xharness output for non-browser cases - (string testCommand, string xharnessArgs, bool useWasmConsoleOutput) = host switch - { - RunHost.V8 => ("wasm test", $"--js-file={jsRelativePath} --engine=V8 -v trace", true), - RunHost.NodeJS => ("wasm test", $"--js-file={jsRelativePath} --engine=NodeJS -v trace", true), - _ => ("wasm test-browser", $"-v trace -b {host} --web-server-use-cop", false) - }; + string testCommand = hostRunner.GetTestCommand(); + XHarnessArgsOptions options = new XHarnessArgsOptions(jsRelativePath, environmentLocale, host); + string xharnessArgs = s_isWindows ? hostRunner.GetXharnessArgsWindowsOS(options) : hostRunner.GetXharnessArgsOtherOS(options); + bool useWasmConsoleOutput = hostRunner.UseWasmConsoleOutput(); extraXHarnessArgs += " " + xharnessArgs; @@ -336,7 +342,7 @@ protected static BuildArgs ExpandBuildArgs(BuildArgs buildArgs, string extraProp if (buildArgs.AOT) { extraProperties = $"{extraProperties}\ntrue"; - extraProperties += $"\n{RuntimeInformation.IsOSPlatform(OSPlatform.Windows)}\n"; + extraProperties += $"\n{s_isWindows}\n"; } if (UseWebcil) { @@ -429,7 +435,8 @@ protected static BuildArgs ExpandBuildArgs(BuildArgs buildArgs, string extraProp options.MainJS ?? "test-main.js", options.HasV8Script, options.TargetFramework ?? DefaultTargetFramework, - options.HasIcudt, + options.GlobalizationMode, + options.PredefinedIcudt ?? "", options.DotnetWasmFromRuntimePack ?? !buildArgs.AOT, UseWebcil); } @@ -633,7 +640,8 @@ protected static void AssertBasicAppBundle(string bundleDir, string mainJS, bool hasV8Script, string targetFramework, - bool hasIcudt = true, + GlobalizationMode? globalizationMode, + string predefinedIcudt = "", bool dotnetWasmFromRuntimePack = true, bool useWebcil = true) { @@ -648,7 +656,7 @@ protected static void AssertBasicAppBundle(string bundleDir, }); AssertFilesExist(bundleDir, new[] { "run-v8.sh" }, expectToExist: hasV8Script); - AssertFilesExist(bundleDir, new[] { "icudt.dat" }, expectToExist: hasIcudt); + AssertIcuAssets(); string managedDir = Path.Combine(bundleDir, "managed"); string bundledMainAppAssembly = @@ -670,6 +678,53 @@ protected static void AssertBasicAppBundle(string bundleDir, } AssertDotNetWasmJs(bundleDir, fromRuntimePack: dotnetWasmFromRuntimePack, targetFramework); + + void AssertIcuAssets() + { + bool expectEFIGS = false; + bool expectCJK = false; + bool expectNOCJK = false; + bool expectFULL = false; + switch (globalizationMode) + { + case GlobalizationMode.Invariant: + break; + case GlobalizationMode.FullIcu: + expectFULL = true; + break; + case GlobalizationMode.PredefinedIcu: + if (string.IsNullOrEmpty(predefinedIcudt)) + throw new ArgumentException("WasmBuildTest is invalid, value for predefinedIcudt is required when GlobalizationMode=PredefinedIcu."); + AssertFilesExist(bundleDir, new[] { predefinedIcudt }, expectToExist: true); + // predefined ICU name can be identical with the icu files from runtime pack + switch (predefinedIcudt) + { + case "icudt.dat": + expectFULL = true; + break; + case "icudt_EFIGS.dat": + expectEFIGS = true; + break; + case "icudt_CJK.dat": + expectCJK = true; + break; + case "icudt_no_CJK.dat": + expectNOCJK = true; + break; + } + break; + default: + // icu shard chosen based on the locale + expectCJK = true; + expectEFIGS = true; + expectNOCJK = true; + break; + } + AssertFilesExist(bundleDir, new[] { "icudt.dat" }, expectToExist: expectFULL); + AssertFilesExist(bundleDir, new[] { "icudt_EFIGS.dat" }, expectToExist: expectEFIGS); + AssertFilesExist(bundleDir, new[] { "icudt_CJK.dat" }, expectToExist: expectCJK); + AssertFilesExist(bundleDir, new[] { "icudt_no_CJK.dat" }, expectToExist: expectNOCJK); + } } protected static void AssertDotNetWasmJs(string bundleDir, bool fromRuntimePack, string targetFramework) @@ -1093,6 +1148,13 @@ public static int Main() return 42; } }"; + + private IHostRunner GetHostRunnerFromRunHost(RunHost host) => host switch + { + RunHost.V8 => new V8HostRunner(), + RunHost.NodeJS => new NodeJSHostRunner(), + _ => new BrowserHostRunner(), + }; } public record BuildArgs(string ProjectName, @@ -1106,20 +1168,21 @@ internal record BuildPaths(string ObjWasmDir, string ObjDir, string BinDir, stri public record BuildProjectOptions ( - Action? InitProject = null, - bool? DotnetWasmFromRuntimePack = null, - bool HasIcudt = true, - bool UseCache = true, - bool ExpectSuccess = true, - bool AssertAppBundle = true, - bool CreateProject = true, - bool Publish = true, - bool BuildOnlyAfterPublish = true, - bool HasV8Script = true, - string? Verbosity = null, - string? Label = null, - string? TargetFramework = null, - string? MainJS = null, + Action? InitProject = null, + bool? DotnetWasmFromRuntimePack = null, + GlobalizationMode? GlobalizationMode = null, + string? PredefinedIcudt = null, + bool UseCache = true, + bool ExpectSuccess = true, + bool AssertAppBundle = true, + bool CreateProject = true, + bool Publish = true, + bool BuildOnlyAfterPublish = true, + bool HasV8Script = true, + string? Verbosity = null, + string? Label = null, + string? TargetFramework = null, + string? MainJS = null, IDictionary? ExtraBuildEnvironmentVariables = null ); @@ -1132,5 +1195,12 @@ public record BlazorBuildOptions bool WarnAsError = true ); + public enum GlobalizationMode + { + Invariant, // no icu + FullIcu, // full icu data: icudt.dat is loaded + PredefinedIcu // user set WasmIcuDataFileName value and we are loading that file + }; + public enum NativeFilesType { FromRuntimePack, Relinked, AOT }; } diff --git a/src/mono/wasm/Wasm.Build.Tests/HostRunner/BrowserHostRunner.cs b/src/mono/wasm/Wasm.Build.Tests/HostRunner/BrowserHostRunner.cs new file mode 100644 index 00000000000000..288048616f6d7a --- /dev/null +++ b/src/mono/wasm/Wasm.Build.Tests/HostRunner/BrowserHostRunner.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#nullable enable + +namespace Wasm.Build.Tests; + +public class BrowserHostRunner : IHostRunner +{ + public string GetTestCommand() => "wasm test-browser"; + public string GetXharnessArgsWindowsOS(XHarnessArgsOptions options) => $"-v trace -b {options.host} --browser-arg=--lang={options.environmentLocale} --web-server-use-cop"; // Windows: chrome.exe --lang=locale + public string GetXharnessArgsOtherOS(XHarnessArgsOptions options) => $"-v trace -b {options.host} --locale={options.environmentLocale} --web-server-use-cop"; // Linux: LANGUAGE=locale ./chrome + public bool UseWasmConsoleOutput() => false; + public bool CanRunWBT() => true; +} diff --git a/src/mono/wasm/Wasm.Build.Tests/HostRunner/IHostRunner.cs b/src/mono/wasm/Wasm.Build.Tests/HostRunner/IHostRunner.cs new file mode 100644 index 00000000000000..0c93a7310ded4a --- /dev/null +++ b/src/mono/wasm/Wasm.Build.Tests/HostRunner/IHostRunner.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#nullable enable + +namespace Wasm.Build.Tests; + +public record XHarnessArgsOptions(string jsRelativePath, string environmentLocale, RunHost host); + +interface IHostRunner +{ + string GetTestCommand(); + string GetXharnessArgsWindowsOS(XHarnessArgsOptions options); + string GetXharnessArgsOtherOS(XHarnessArgsOptions options); + bool UseWasmConsoleOutput(); + bool CanRunWBT(); +} diff --git a/src/mono/wasm/Wasm.Build.Tests/HostRunner/NodeJSHostRunner.cs b/src/mono/wasm/Wasm.Build.Tests/HostRunner/NodeJSHostRunner.cs new file mode 100644 index 00000000000000..cf311557c27f2e --- /dev/null +++ b/src/mono/wasm/Wasm.Build.Tests/HostRunner/NodeJSHostRunner.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#nullable enable + +namespace Wasm.Build.Tests; + +public class NodeJSHostRunner : IHostRunner +{ + public string GetTestCommand() => "wasm test"; + public string GetXharnessArgsWindowsOS(XHarnessArgsOptions options) => $"--js-file={options.jsRelativePath} --engine=NodeJS -v trace"; + public string GetXharnessArgsOtherOS(XHarnessArgsOptions options) => $"--js-file={options.jsRelativePath} --engine=NodeJS -v trace --locale={options.environmentLocale}"; + public bool UseWasmConsoleOutput() => true; + public bool CanRunWBT() => true; +} diff --git a/src/mono/wasm/Wasm.Build.Tests/HostRunner/V8HostRunner.cs b/src/mono/wasm/Wasm.Build.Tests/HostRunner/V8HostRunner.cs new file mode 100644 index 00000000000000..885aba29ad4634 --- /dev/null +++ b/src/mono/wasm/Wasm.Build.Tests/HostRunner/V8HostRunner.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#nullable enable + +namespace Wasm.Build.Tests; + +using System.Runtime.InteropServices; + +public class V8HostRunner : IHostRunner +{ + private string GetXharnessArgs(string jsRelativePath) => $"--js-file={jsRelativePath} --engine=V8 -v trace"; + + public string GetTestCommand() => "wasm test"; + public string GetXharnessArgsWindowsOS(XHarnessArgsOptions options) => GetXharnessArgs(options.jsRelativePath); + public string GetXharnessArgsOtherOS(XHarnessArgsOptions options) => GetXharnessArgs(options.jsRelativePath); + public bool UseWasmConsoleOutput() => true; + public bool CanRunWBT() => !RuntimeInformation.IsOSPlatform(OSPlatform.Windows); +} diff --git a/src/mono/wasm/Wasm.Build.Tests/IcuShardingTests.cs b/src/mono/wasm/Wasm.Build.Tests/IcuShardingTests.cs new file mode 100644 index 00000000000000..19a4ac63434683 --- /dev/null +++ b/src/mono/wasm/Wasm.Build.Tests/IcuShardingTests.cs @@ -0,0 +1,276 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; +using System.Collections.Generic; + +#nullable enable + +namespace Wasm.Build.Tests; + +public class IcuShardingTests : BuildTestBase +{ + public IcuShardingTests(ITestOutputHelper output, SharedBuildPerTestClassFixture buildContext) + : base(output, buildContext) { } + + // custom file contains only locales "cy-GB", "is-IS", "bs-BA", "lb-LU" and fallback locale: "en-US": + private static string s_customIcuPath = Path.Combine(BuildEnvironment.TestAssetsPath, "icudt_custom.dat"); + public record SundayNames { + public static string English = "Sunday"; + public static string French = "dimanche"; + public static string Spanish = "domingo"; + public static string Chinese = "星期日"; + public static string Japanese = "日曜日"; + public static string Slovak = "nedeľa"; + } + + private const string FallbackSundayNameEnUS = "Sunday"; + + private static readonly string s_customIcuTestedLocales = $@"new Locale[] {{ + new Locale(""cy-GB"", ""Dydd Sul""), new Locale(""is-IS"", ""sunnudagur""), new Locale(""bs-BA"", ""nedjelja""), new Locale(""lb-LU"", ""Sonndeg""), + new Locale(""fr-FR"", null), new Locale(""hr-HR"", null), new Locale(""ko-KR"", null) + }}"; + private static string GetEfigsTestedLocales(string fallbackSundayName=FallbackSundayNameEnUS) => $@"new Locale[] {{ + new Locale(""en-US"", ""{SundayNames.English}""), new Locale(""fr-FR"", ""{SundayNames.French}""), new Locale(""es-ES"", ""{SundayNames.Spanish}""), + new Locale(""pl-PL"", ""{fallbackSundayName}""), new Locale(""ko-KR"", ""{fallbackSundayName}""), new Locale(""cs-CZ"", ""{fallbackSundayName}"") + }}"; + private static string GetCjkTestedLocales(string fallbackSundayName=FallbackSundayNameEnUS) => $@"new Locale[] {{ + new Locale(""en-GB"", ""{SundayNames.English}""), new Locale(""zh-CN"", ""{SundayNames.Chinese}""), new Locale(""ja-JP"", ""{SundayNames.Japanese}""), + new Locale(""fr-FR"", ""{fallbackSundayName}""), new Locale(""hr-HR"", ""{fallbackSundayName}""), new Locale(""it-IT"", ""{fallbackSundayName}"") + }}"; + private static string GetNocjkTestedLocales(string fallbackSundayName=FallbackSundayNameEnUS) => $@"new Locale[] {{ + new Locale(""en-AU"", ""{SundayNames.English}""), new Locale(""fr-FR"", ""{SundayNames.French}""), new Locale(""sk-SK"", ""{SundayNames.Slovak}""), + new Locale(""ja-JP"", ""{fallbackSundayName}""), new Locale(""ko-KR"", ""{fallbackSundayName}""), new Locale(""zh-CN"", ""{fallbackSundayName}"") + }}"; + private static readonly string s_fullIcuTestedLocales = $@"new Locale[] {{ + new Locale(""en-GB"", ""{SundayNames.English}""), new Locale(""sk-SK"", ""{SundayNames.Slovak}""), new Locale(""zh-CN"", ""{SundayNames.Chinese}"") + }}"; + + public static IEnumerable IcuExpectedAndMissingCustomShardTestData(bool aot, RunHost host) + => ConfigWithAOTData(aot) + .Multiply( + new object[] { s_customIcuPath, s_customIcuTestedLocales, false }, + new object[] { s_customIcuPath, s_customIcuTestedLocales, true }) + .WithRunHosts(host) + .UnwrapItemsAsArrays(); + + public static IEnumerable IcuExpectedAndMissingShardFromRuntimePackTestData(bool aot, RunHost host) + => ConfigWithAOTData(aot) + .Multiply( + new object[] { "icudt.dat", + $@"new Locale[] {{ + new Locale(""en-GB"", ""{SundayNames.English}""), new Locale(""zh-CN"", ""{SundayNames.Chinese}""), new Locale(""sk-SK"", ""{SundayNames.Slovak}""), + new Locale(""xx-yy"", null) }}" }, + new object[] { "icudt_EFIGS.dat", GetEfigsTestedLocales() }, + new object[] { "icudt_CJK.dat", GetCjkTestedLocales() }, + new object[] { "icudt_no_CJK.dat", GetNocjkTestedLocales() }) + .WithRunHosts(host) + .UnwrapItemsAsArrays(); + + public static IEnumerable IcuExpectedAndMissingAutomaticShardTestData(bool aot) + => ConfigWithAOTData(aot) + .Multiply( + new object[] { "fr-FR", GetEfigsTestedLocales(SundayNames.French)}, + new object[] { "ja-JP", GetCjkTestedLocales(SundayNames.Japanese) }, + new object[] { "sk-SK", GetNocjkTestedLocales(SundayNames.Slovak) }) + .WithRunHosts(BuildTestBase.s_hostsForOSLocaleSensitiveTests) + .UnwrapItemsAsArrays(); + + public static IEnumerable FullIcuWithInvariantTestData(bool aot, RunHost host) + => ConfigWithAOTData(aot) + .Multiply( + // in invariant mode, all locales should be missing + new object[] { true, true, "Array.Empty()" }, + new object[] { true, false, "Array.Empty()" }, + new object[] { false, false, GetEfigsTestedLocales() }, + new object[] { false, true, s_fullIcuTestedLocales}) + .WithRunHosts(host) + .UnwrapItemsAsArrays(); + + public static IEnumerable FullIcuWithICustomIcuTestData(bool aot, RunHost host) + => ConfigWithAOTData(aot) + .Multiply( + new object[] { true }, + new object[] { false }) + .WithRunHosts(host) + .UnwrapItemsAsArrays(); + + private static string GetProgramText(string testedLocales, bool onlyPredefinedCultures=false, string fallbackSundayName=FallbackSundayNameEnUS) => $@" + #nullable enable + + using System; + using System.Globalization; + + Console.WriteLine($""Current culture: '{{CultureInfo.CurrentCulture.Name}}'""); + + string fallbackSundayName = ""{fallbackSundayName}""; + bool onlyPredefinedCultures = {(onlyPredefinedCultures ? "true" : "false")}; + Locale[] localesToTest = {testedLocales}; + + bool fail = false; + foreach (var testLocale in localesToTest) + {{ + bool expectMissing = string.IsNullOrEmpty(testLocale.SundayName); + bool ctorShouldFail = expectMissing && onlyPredefinedCultures; + CultureInfo culture; + + try + {{ + culture = new CultureInfo(testLocale.Code); + if (ctorShouldFail) + {{ + Console.WriteLine($""CultureInfo..ctor did not throw an exception for {{testLocale.Code}} as was expected.""); + fail = true; + continue; + }} + }} + catch(CultureNotFoundException cnfe) when (ctorShouldFail && cnfe.Message.Contains($""{{testLocale.Code}} is an invalid culture identifier."")) + {{ + Console.WriteLine($""{{testLocale.Code}}: Success. .ctor failed as expected.""); + continue; + }} + + string expectedSundayName = (expectMissing && !onlyPredefinedCultures) + ? fallbackSundayName + : testLocale.SundayName; + var actualLocalizedSundayName = culture.DateTimeFormat.GetDayName(new DateTime(2000,01,02).DayOfWeek); + if (expectedSundayName != actualLocalizedSundayName) + {{ + Console.WriteLine($""Error: incorrect localized value for Sunday in locale {{testLocale.Code}}. Expected '{{expectedSundayName}}' but got '{{actualLocalizedSundayName}}'.""); + fail = true; + continue; + }} + Console.WriteLine($""{{testLocale.Code}}: Success. Sunday name: {{actualLocalizedSundayName}}""); + }} + return fail ? -1 : 42; + + public record Locale(string Code, string? SundayName); + "; + + private void TestIcuShards(BuildArgs buildArgs, string shardName, string testedLocales, RunHost host, string id, bool onlyPredefinedCultures=false) + { + string projectName = $"shard_{Path.GetFileName(shardName)}_{buildArgs.Config}_{buildArgs.AOT}"; + bool dotnetWasmFromRuntimePack = !(buildArgs.AOT || buildArgs.Config == "Release"); + + buildArgs = buildArgs with { ProjectName = projectName }; + string extraProperties = onlyPredefinedCultures ? + $"{shardName}true" : + $"{shardName}"; + buildArgs = ExpandBuildArgs(buildArgs, extraProperties: extraProperties); + + string programText = GetProgramText(testedLocales, onlyPredefinedCultures); + _testOutput.WriteLine($"----- Program: -----{Environment.NewLine}{programText}{Environment.NewLine}-------"); + (_, string output) = BuildProject(buildArgs, + id: id, + new BuildProjectOptions( + InitProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText), + DotnetWasmFromRuntimePack: dotnetWasmFromRuntimePack, + GlobalizationMode: GlobalizationMode.PredefinedIcu, + PredefinedIcudt: shardName)); + + string runOutput = RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); + } + + [Theory] + [MemberData(nameof(IcuExpectedAndMissingCustomShardTestData), parameters: new object[] { false, RunHost.NodeJS | RunHost.Chrome })] + [MemberData(nameof(IcuExpectedAndMissingCustomShardTestData), parameters: new object[] { true, RunHost.NodeJS | RunHost.Chrome })] + public void CustomIcuShard(BuildArgs buildArgs, string shardName, string testedLocales, bool onlyPredefinedCultures, RunHost host, string id) => + TestIcuShards(buildArgs, shardName, testedLocales, host, id, onlyPredefinedCultures); + + [Theory] + [MemberData(nameof(IcuExpectedAndMissingShardFromRuntimePackTestData), parameters: new object[] { false,RunHost.NodeJS | RunHost.Chrome })] + [MemberData(nameof(IcuExpectedAndMissingShardFromRuntimePackTestData), parameters: new object[] { true, RunHost.NodeJS | RunHost.Chrome })] + public void DefaultAvailableIcuShardsFromRuntimePack(BuildArgs buildArgs, string shardName, string testedLocales, RunHost host, string id) => + TestIcuShards(buildArgs, shardName, testedLocales, host, id); + + [Theory] + [MemberData(nameof(IcuExpectedAndMissingAutomaticShardTestData), parameters: new object[] { false })] + [MemberData(nameof(IcuExpectedAndMissingAutomaticShardTestData), parameters: new object[] { true })] + public void AutomaticShardSelectionDependingOnEnvLocale(BuildArgs buildArgs, string environmentLocale, string testedLocales, RunHost host, string id) + { + string projectName = $"automatic_shard_{environmentLocale}_{buildArgs.Config}_{buildArgs.AOT}"; + bool dotnetWasmFromRuntimePack = !(buildArgs.AOT || buildArgs.Config == "Release"); + + buildArgs = buildArgs with { ProjectName = projectName }; + buildArgs = ExpandBuildArgs(buildArgs); + + string programText = GetProgramText(testedLocales); + _testOutput.WriteLine($"----- Program: -----{Environment.NewLine}{programText}{Environment.NewLine}-------"); + (_, string output) = BuildProject(buildArgs, + id: id, + new BuildProjectOptions( + InitProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText), + DotnetWasmFromRuntimePack: dotnetWasmFromRuntimePack)); + string runOutput = RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id, environmentLocale: environmentLocale); + } + + [Theory] + [MemberData(nameof(FullIcuWithInvariantTestData), parameters: new object[] { false, RunHost.NodeJS | RunHost.Chrome })] + [MemberData(nameof(FullIcuWithInvariantTestData), parameters: new object[] { true, RunHost.NodeJS | RunHost.Chrome })] + public void FullIcuFromRuntimePackWithInvariant(BuildArgs buildArgs, bool invariant, bool fullIcu, string testedLocales, RunHost host, string id) + { + string projectName = $"fullIcuInvariant_{fullIcu}_{invariant}_{buildArgs.Config}_{buildArgs.AOT}"; + bool dotnetWasmFromRuntimePack = !(buildArgs.AOT || buildArgs.Config == "Release"); + + buildArgs = buildArgs with { ProjectName = projectName }; + buildArgs = ExpandBuildArgs(buildArgs, extraProperties: $"{invariant}{fullIcu}"); + + string programText = GetProgramText(testedLocales); + _testOutput.WriteLine($"----- Program: -----{Environment.NewLine}{programText}{Environment.NewLine}-------"); + (_, string output) = BuildProject(buildArgs, + id: id, + new BuildProjectOptions( + InitProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText), + DotnetWasmFromRuntimePack: dotnetWasmFromRuntimePack, + GlobalizationMode: invariant ? GlobalizationMode.Invariant : fullIcu ? GlobalizationMode.FullIcu : null)); + + string runOutput = RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); + } + + [Theory] + [MemberData(nameof(FullIcuWithICustomIcuTestData), parameters: new object[] { false, RunHost.NodeJS | RunHost.Chrome })] + [MemberData(nameof(FullIcuWithICustomIcuTestData), parameters: new object[] { true, RunHost.NodeJS | RunHost.Chrome })] + public void FullIcuFromRuntimePackWithCustomIcu(BuildArgs buildArgs, bool fullIcu, RunHost host, string id) + { + string projectName = $"fullIcuCustom_{fullIcu}_{buildArgs.Config}_{buildArgs.AOT}"; + bool dotnetWasmFromRuntimePack = !(buildArgs.AOT || buildArgs.Config == "Release"); + + buildArgs = buildArgs with { ProjectName = projectName }; + buildArgs = ExpandBuildArgs(buildArgs, extraProperties: $"{s_customIcuPath}{fullIcu}"); + + string testedLocales = fullIcu ? s_fullIcuTestedLocales : s_customIcuTestedLocales; + string programText = GetProgramText(testedLocales); + _testOutput.WriteLine($"----- Program: -----{Environment.NewLine}{programText}{Environment.NewLine}-------"); + (_, string output) = BuildProject(buildArgs, + id: id, + new BuildProjectOptions( + InitProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText), + DotnetWasmFromRuntimePack: dotnetWasmFromRuntimePack, + GlobalizationMode: fullIcu ? GlobalizationMode.FullIcu : GlobalizationMode.PredefinedIcu, + PredefinedIcudt: fullIcu ? "" : s_customIcuPath)); + if (fullIcu) + Assert.Contains("$(WasmIcuDataFileName) has no effect when $(WasmIncludeFullIcuData) is set to true.", output); + + string runOutput = RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); + } + + [Theory] + [BuildAndRun(host: RunHost.None)] + public void NonExistingCustomFileAssertError(BuildArgs buildArgs, string id) + { + string projectName = $"invalidCustomIcu_{buildArgs.Config}_{buildArgs.AOT}"; + buildArgs = buildArgs with { ProjectName = projectName }; + buildArgs = ExpandBuildArgs(buildArgs, extraProperties: $"nonexisting.dat"); + + (_, string output) = BuildProject(buildArgs, + id: id, + new BuildProjectOptions( + InitProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), + ExpectSuccess: false)); + Assert.Contains("File in location $(WasmIcuDataFileName)=nonexisting.dat cannot be found neither when used as absolute path nor a relative runtime pack path.", output); + } +} diff --git a/src/mono/wasm/Wasm.Build.Tests/InvariantGlobalizationTests.cs b/src/mono/wasm/Wasm.Build.Tests/InvariantGlobalizationTests.cs index f34cb9cde7d655..41dac50141cf7c 100644 --- a/src/mono/wasm/Wasm.Build.Tests/InvariantGlobalizationTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/InvariantGlobalizationTests.cs @@ -78,7 +78,7 @@ private void TestInvariantGlobalization(BuildArgs buildArgs, bool? invariantGlob new BuildProjectOptions( InitProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText), DotnetWasmFromRuntimePack: dotnetWasmFromRuntimePack, - HasIcudt: invariantGlobalization == null || invariantGlobalization.Value == false)); + GlobalizationMode: invariantGlobalization == true ? GlobalizationMode.Invariant : null)); if (invariantGlobalization == true) { diff --git a/src/mono/wasm/Wasm.Build.Tests/NativeRebuildTests/NativeRebuildTestsBase.cs b/src/mono/wasm/Wasm.Build.Tests/NativeRebuildTests/NativeRebuildTestsBase.cs index 361f9bfd270ce2..c9b4e2bc666020 100644 --- a/src/mono/wasm/Wasm.Build.Tests/NativeRebuildTests/NativeRebuildTestsBase.cs +++ b/src/mono/wasm/Wasm.Build.Tests/NativeRebuildTests/NativeRebuildTestsBase.cs @@ -52,7 +52,7 @@ public NativeRebuildTestsBase(ITestOutputHelper output, SharedBuildPerTestClassF new BuildProjectOptions( InitProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText), DotnetWasmFromRuntimePack: false, - HasIcudt: !invariant, + GlobalizationMode: invariant ? GlobalizationMode.Invariant : null, CreateProject: true)); RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: RunHost.Chrome, id: id); @@ -82,7 +82,7 @@ protected string Rebuild(bool nativeRelink, bool invariant, BuildArgs buildArgs, id: id, new BuildProjectOptions( DotnetWasmFromRuntimePack: false, - HasIcudt: !invariant, + GlobalizationMode: invariant ? GlobalizationMode.Invariant : null, CreateProject: false, UseCache: false, Verbosity: verbosity)); diff --git a/src/mono/wasm/build/WasmApp.targets b/src/mono/wasm/build/WasmApp.targets index 6b68828e6685b3..d59b265dd91818 100644 --- a/src/mono/wasm/build/WasmApp.targets +++ b/src/mono/wasm/build/WasmApp.targets @@ -69,6 +69,8 @@ - $(WasmEnableExceptionHandling) - Enable support for the WASM Exception Handling feature. - $(WasmEnableSIMD) - Enable support for the WASM SIMD feature. - $(WasmEnableWebcil) - Enable conversion of assembly .dlls to .webcil + - $(WasmIncludeFullIcuData) - Loads full ICU data (icudt.dat). Defaults to false. Only applicable when InvariantGlobalization=false. + - $(WasmIcuDataFileName) - Name/path of ICU globalization file loaded to app. Only when InvariantGloblization=false and WasmIncludeFullIcuData=false. - $(WasmAllowUndefinedSymbols) - Controls whether undefined symbols are allowed or not, if true, appends 'allow-undefined' and sets 'ERROR_ON_UNDEFINED_SYMBOLS=0' as arguments for wasm-ld, if false (default), removes 'allow-undefined' and sets 'ERROR_ON_UNDEFINED_SYMBOLS=1'. @@ -316,12 +318,12 @@ - icudt.dat - <_HasDotnetWasm Condition="'%(WasmNativeAsset.FileName)%(WasmNativeAsset.Extension)' == 'dotnet.wasm'">true <_HasDotnetJsWorker Condition="'%(WasmNativeAsset.FileName)%(WasmNativeAsset.Extension)' == 'dotnet.worker.js'">true <_HasDotnetJsSymbols Condition="'%(WasmNativeAsset.FileName)%(WasmNativeAsset.Extension)' == 'dotnet.js.symbols'">true <_HasDotnetJs Condition="'%(WasmNativeAsset.FileName)%(WasmNativeAsset.Extension)' == 'dotnet.js'">true + <_WasmIcuDataFileName Condition="'$(WasmIcuDataFileName)' != '' and Exists('$(WasmIcuDataFileName)')">$(WasmIcuDataFileName) + <_WasmIcuDataFileName Condition="'$(WasmIcuDataFileName)' != '' and !Exists('$(WasmIcuDataFileName)')">$(MicrosoftNetCoreAppRuntimePackRidNativeDir)$(WasmIcuDataFileName) @@ -333,20 +335,27 @@ Condition="'$(WasmEmitSymbolMap)' == 'true' and '$(_HasDotnetJsSymbols)' != 'true' and Exists('$(MicrosoftNetCoreAppRuntimePackRidNativeDir)dotnet.js.symbols')" /> - - - + + + <_IcuAvailableDataFiles Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)icudt_*" /> + + + + + + + <_WasmAppIncludeThreadsWorker Condition="'$(WasmEnableThreads)' == 'true' or '$(WasmEnablePerfTracing)' == 'true'">true @@ -367,7 +376,7 @@ InvariantGlobalization="$(InvariantGlobalization)" SatelliteAssemblies="@(_WasmSatelliteAssemblies)" FilesToIncludeInFileSystem="@(WasmFilesToIncludeInFileSystem)" - IcuDataFileName="$(WasmIcuDataFileName)" + IcuDataFileNames="@(WasmIcuDataFileNames)" RemoteSources="@(WasmRemoteSources)" ExtraFilesToDeploy="@(WasmExtraFilesToDeploy)" ExtraConfig="@(WasmExtraConfig)" diff --git a/src/mono/wasm/debugger/DebuggerTestSuite/ChromeProvider.cs b/src/mono/wasm/debugger/DebuggerTestSuite/ChromeProvider.cs index fc7878af3bf689..7d95f72e675185 100644 --- a/src/mono/wasm/debugger/DebuggerTestSuite/ChromeProvider.cs +++ b/src/mono/wasm/debugger/DebuggerTestSuite/ChromeProvider.cs @@ -47,12 +47,16 @@ public async Task StartBrowserAndProxyAsync(HttpContext context, string messagePrefix, ILoggerFactory loggerFactory, CancellationTokenSource cts, - int browserReadyTimeoutMs = 20000) + int browserReadyTimeoutMs = 20000, + string locale = "en-US") { string? line; try { - ProcessStartInfo psi = GetProcessStartInfo(s_browserPath.Value, GetInitParms(remoteDebuggingPort), targetUrl); + // for WIndows setting --lang arg is enough + if (!OperatingSystem.IsWindows()) + Environment.SetEnvironmentVariable("LANGUAGE", locale); + ProcessStartInfo psi = GetProcessStartInfo(s_browserPath.Value, GetInitParms(remoteDebuggingPort, locale), targetUrl); line = await LaunchHostAsync( psi, context, @@ -161,9 +165,9 @@ private async Task ExtractConnUrl (string str, ILogger logger) return wsURl; } - private static string GetInitParms(int port) + private static string GetInitParms(int port, string lang="en-US") { - string str = $"--headless --disable-gpu --lang=en-US --incognito --remote-debugging-port={port}"; + string str = $"--headless --disable-gpu --lang={lang} --incognito --remote-debugging-port={port}"; if (File.Exists("/.dockerenv")) { Console.WriteLine ("Detected a container, disabling sandboxing for debugger tests."); diff --git a/src/mono/wasm/debugger/DebuggerTestSuite/DateTimeTests.cs b/src/mono/wasm/debugger/DebuggerTestSuite/DateTimeTests.cs index 31cb68cf4f62fa..b6a7e45e0091be 100644 --- a/src/mono/wasm/debugger/DebuggerTestSuite/DateTimeTests.cs +++ b/src/mono/wasm/debugger/DebuggerTestSuite/DateTimeTests.cs @@ -9,14 +9,50 @@ namespace DebuggerTests { - public class DateTimeTests : DebuggerTests + public class DateTimeTestsCJK : DebuggerTests { - public DateTimeTests(ITestOutputHelper testOutput) : base(testOutput) + public DateTimeTestsCJK(ITestOutputHelper testOutput) : base(testOutput, locale: "ja-JA") + {} + + [ConditionalTheory(nameof(RunningOnChrome))] + [InlineData("ja-JP", "yyyy\u5E74M\u6708d\u65E5dddd H:mm:ss", "yyyy\u5E74M\u6708d\u65E5dddd", "H:mm:ss", "yyyy/MM/dd", "H:mm")] + public async Task CheckDateTimeLocale(string locale, string fdtp, string ldp, string ltp, string sdp, string stp) + { + var debugger_test_loc = "dotnet://debugger-test.dll/debugger-datetime-test.cs"; + + await SetBreakpointInMethod("debugger-test", "DebuggerTests.DateTimeTest", "LocaleTest", 15); + + var pause_location = await EvaluateAndCheck( + "window.setTimeout(function() { invoke_static_method ('[debugger-test] DebuggerTests.DateTimeTest:LocaleTest'," + + $"'{locale}'); }}, 1);", + debugger_test_loc, 25, 12, "DebuggerTests.DateTimeTest.LocaleTest", + locals_fn: async (locals) => + { + DateTimeFormatInfo dtfi = CultureInfo.GetCultureInfo(locale).DateTimeFormat; + CultureInfo.CurrentCulture = new CultureInfo(locale, false); + + await CheckProps(locals, new + { + fdtp = TString(fdtp), + ldp = TString(ldp), + ltp = TString(ltp), + sdp = TString(sdp), + stp = TString(stp), + dt = TDateTime(new DateTime(2020, 1, 2, 3, 4, 5)) + }, "locals", num_fields: 8); + } + ); + } + + } + + public class DateTimeTestsEFIGS : DebuggerTests + { + public DateTimeTestsEFIGS(ITestOutputHelper testOutput) : base(testOutput) {} [Theory] [InlineData("en-US", "dddd, MMMM d, yyyy h:mm:ss tt", "dddd, MMMM d, yyyy", "h:mm:ss tt", "M/d/yyyy", "h:mm tt")] - [InlineData("ja-JP", "yyyy\u5E74M\u6708d\u65E5dddd H:mm:ss", "yyyy\u5E74M\u6708d\u65E5dddd", "H:mm:ss", "yyyy/MM/dd", "H:mm")] [InlineData("es-ES", "dddd, d 'de' MMMM 'de' yyyy H:mm:ss", "dddd, d 'de' MMMM 'de' yyyy", "H:mm:ss", "d/M/yyyy", "H:mm")] [InlineData("de-DE", "dddd, d. MMMM yyyy HH:mm:ss", "dddd, d. MMMM yyyy", "HH:mm:ss", "dd.MM.yyyy", "HH:mm")] public async Task CheckDateTimeLocale(string locale, string fdtp, string ldp, string ltp, string sdp, string stp) diff --git a/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestBase.cs b/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestBase.cs index 73a6aa0f6aa0d9..225fcabff37f6c 100644 --- a/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestBase.cs +++ b/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestBase.cs @@ -24,8 +24,8 @@ public class DebuggerTests : DebuggerTestFirefox #endif { - public DebuggerTests(ITestOutputHelper testOutput, string driver = "debugger-driver.html") - : base(testOutput, driver) + public DebuggerTests(ITestOutputHelper testOutput, string locale = "en-US", string driver = "debugger-driver.html") + : base(testOutput, locale, driver) {} } @@ -130,19 +130,19 @@ static DebuggerTestBase() Directory.Delete(TempPath, recursive: true); } - public DebuggerTestBase(ITestOutputHelper testOutput, string driver = "debugger-driver.html") + public DebuggerTestBase(ITestOutputHelper testOutput, string locale, string driver = "debugger-driver.html") { _env = new TestEnvironment(testOutput); _testOutput = testOutput; Id = Interlocked.Increment(ref s_idCounter); // the debugger is working in locale of the debugged application. For example Datetime.ToString() // we want the test to mach it. We are also starting chrome with --lang=en-US - System.Globalization.CultureInfo.CurrentCulture = new System.Globalization.CultureInfo("en-US"); + System.Globalization.CultureInfo.CurrentCulture = new System.Globalization.CultureInfo(locale); insp = new Inspector(Id, _testOutput); cli = insp.Client; scripts = SubscribeToScripts(insp); - startTask = TestHarnessProxy.Start(DebuggerTestAppPath, driver, UrlToRemoteDebugging(), testOutput); + startTask = TestHarnessProxy.Start(DebuggerTestAppPath, driver, UrlToRemoteDebugging(), testOutput, locale); } public virtual async Task InitializeAsync() diff --git a/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestFirefox.cs b/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestFirefox.cs index 5e8e15fb826fc6..282ecbf9a56645 100644 --- a/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestFirefox.cs +++ b/src/mono/wasm/debugger/DebuggerTestSuite/DebuggerTestFirefox.cs @@ -16,7 +16,8 @@ public class DebuggerTestFirefox : DebuggerTestBase { private new TimeSpan TestTimeout => base.TestTimeout * 5; internal FirefoxInspectorClient _client; - public DebuggerTestFirefox(ITestOutputHelper testOutput, string driver = "debugger-driver.html"):base(testOutput, driver) + public DebuggerTestFirefox(ITestOutputHelper testOutput, string driver = "debugger-driver.html", string locale = "en-US") + : base(testOutput, driver, locale) { if (insp.Client is not FirefoxInspectorClient) throw new Exception($"Bug: client should be {nameof(FirefoxInspectorClient)} for use with {nameof(DebuggerTestFirefox)}"); diff --git a/src/mono/wasm/debugger/DebuggerTestSuite/FirefoxProvider.cs b/src/mono/wasm/debugger/DebuggerTestSuite/FirefoxProvider.cs index 1744336c5b594b..b64fe84ca3d92f 100644 --- a/src/mono/wasm/debugger/DebuggerTestSuite/FirefoxProvider.cs +++ b/src/mono/wasm/debugger/DebuggerTestSuite/FirefoxProvider.cs @@ -38,14 +38,15 @@ public async Task StartBrowserAndProxyAsync(HttpContext context, string messagePrefix, ILoggerFactory loggerFactory, CancellationTokenSource cts, - int browserReadyTimeoutMs = 20000) + int browserReadyTimeoutMs = 20000, + string locale = "en-US") { if (_isDisposed) throw new ObjectDisposedException(nameof(FirefoxProvider)); try { - string args = $"-profile {GetProfilePath(Id)} -headless -new-instance -private -start-debugger-server {remoteDebuggingPort}"; + string args = $"-profile {GetProfilePath(Id)} -headless -new-instance -private -start-debugger-server {remoteDebuggingPort} -UILocale {locale}"; ProcessStartInfo? psi = GetProcessStartInfo(s_browserPath.Value, args, targetUrl); string? line = await LaunchHostAsync( psi, diff --git a/src/mono/wasm/debugger/DebuggerTestSuite/TestHarnessOptions.cs b/src/mono/wasm/debugger/DebuggerTestSuite/TestHarnessOptions.cs index c22f0563747e74..077e7a378e0866 100644 --- a/src/mono/wasm/debugger/DebuggerTestSuite/TestHarnessOptions.cs +++ b/src/mono/wasm/debugger/DebuggerTestSuite/TestHarnessOptions.cs @@ -17,5 +17,6 @@ public class TestHarnessOptions : ProxyOptions public bool WebServerUseCors { get; set; } public bool WebServerUseCrossOriginPolicy { get; set; } public Func, Task> ExtractConnUrl { get; set; } + public string Locale { get; set; } } } diff --git a/src/mono/wasm/debugger/DebuggerTestSuite/TestHarnessProxy.cs b/src/mono/wasm/debugger/DebuggerTestSuite/TestHarnessProxy.cs index 21cfe68664a5bf..371a8e954d751e 100644 --- a/src/mono/wasm/debugger/DebuggerTestSuite/TestHarnessProxy.cs +++ b/src/mono/wasm/debugger/DebuggerTestSuite/TestHarnessProxy.cs @@ -35,7 +35,7 @@ public class TestHarnessProxy private static readonly ConcurrentDictionary>> s_exitHandlers = new(); private static readonly ConcurrentDictionary s_statusTable = new(); - public static Task Start(string appPath, string pagePath, string url, ITestOutputHelper testOutput) + public static Task Start(string appPath, string pagePath, string url, ITestOutputHelper testOutput, string locale = "en-US") { TestHarnessOptions options = new() { @@ -43,7 +43,8 @@ public static Task Start(string appPath, string pagePath, string url, ITestOutpu PagePath = pagePath, DevToolsUrl = new Uri(url), WebServerUseCors = false, - WebServerUseCrossOriginPolicy = true + WebServerUseCrossOriginPolicy = true, + Locale = locale }; lock (proxyLock) diff --git a/src/mono/wasm/debugger/DebuggerTestSuite/TestHarnessStartup.cs b/src/mono/wasm/debugger/DebuggerTestSuite/TestHarnessStartup.cs index 168c17d4a593b2..55f518fc24e179 100644 --- a/src/mono/wasm/debugger/DebuggerTestSuite/TestHarnessStartup.cs +++ b/src/mono/wasm/debugger/DebuggerTestSuite/TestHarnessStartup.cs @@ -145,7 +145,8 @@ await provider.StartBrowserAndProxyAsync(context, browserPort, message_prefix, _loggerFactory, - cts).ConfigureAwait(false); + cts, + locale: options.Locale).ConfigureAwait(false); } else if (host == WasmHost.Firefox) { @@ -157,7 +158,8 @@ await provider.StartBrowserAndProxyAsync(context, firefox_proxy_port, message_prefix, _loggerFactory, - cts).ConfigureAwait(false); + cts, + locale: options.Locale).ConfigureAwait(false); } Logger.LogDebug($"{message_prefix} TestHarnessStartup done"); } diff --git a/src/mono/wasm/runtime/assets.ts b/src/mono/wasm/runtime/assets.ts index e9b98a8a242da4..c485d0d8d94ce0 100644 --- a/src/mono/wasm/runtime/assets.ts +++ b/src/mono/wasm/runtime/assets.ts @@ -47,6 +47,37 @@ const skipInstantiateByAssetTypes: { "dotnetwasm": true, }; +export function get_preferred_icu_asset(): string | null { + if (!runtimeHelpers.config.assets) + return null; + + // By setting user can define what ICU source file they want to load. + // There is no need to check application's culture when is set. + // If it was not set, then we have 3 "icu" assets in config and we should choose + // only one for loading, the one that matches the application's locale. + const icuAssets = runtimeHelpers.config.assets.filter(a => a["behavior"] == "icu"); + if (icuAssets.length === 1) + return icuAssets[0].name; + + // reads the browsers locale / the OS's locale + const preferredCulture = ENVIRONMENT_IS_WEB ? navigator.language : Intl.DateTimeFormat().resolvedOptions().locale; + const prefix = preferredCulture.split("-")[0]; + const CJK = "icudt_CJK.dat"; + const EFIGS = "icudt_EFIGS.dat"; + const OTHERS = "icudt_no_CJK.dat"; + + // not all "fr-*", "it-*", "de-*", "es-*" are in EFIGS, only the one that is mostly used + if (prefix == "en" || ["fr", "fr-FR", "it", "it-IT", "de", "de-DE", "es", "es-ES"].includes(preferredCulture)) + return EFIGS; + if (["zh", "ko", "ja"].includes(prefix)) + return CJK; + return OTHERS; +} + +export function shouldLoadIcuAsset(asset : AssetEntryInternal, preferredIcuAsset: string | null) : boolean{ + return !(asset.behavior == "icu" && asset.name != preferredIcuAsset); +} + export function resolve_asset_path(behavior: AssetBehaviours) { const asset: AssetEntryInternal | undefined = runtimeHelpers.config.assets?.find(a => a.behavior == behavior); mono_assert(asset, () => `Can't find asset for ${behavior}`); @@ -56,6 +87,7 @@ export function resolve_asset_path(behavior: AssetBehaviours) { return asset; } export async function mono_download_assets(): Promise { + const preferredIcuAsset = get_preferred_icu_asset(); if (runtimeHelpers.diagnosticTracing) console.debug("MONO_WASM: mono_download_assets"); runtimeHelpers.maxParallelDownloads = runtimeHelpers.config.maxParallelDownloads || runtimeHelpers.maxParallelDownloads; runtimeHelpers.enableDownloadRetry = runtimeHelpers.config.enableDownloadRetry || runtimeHelpers.enableDownloadRetry; @@ -70,10 +102,10 @@ export async function mono_download_assets(): Promise { mono_assert(!asset.resolvedUrl || typeof asset.resolvedUrl === "string", "asset resolvedUrl could be string"); mono_assert(!asset.hash || typeof asset.hash === "string", "asset resolvedUrl could be string"); mono_assert(!asset.pendingDownload || typeof asset.pendingDownload === "object", "asset pendingDownload could be object"); - if (!skipInstantiateByAssetTypes[asset.behavior]) { + if (!skipInstantiateByAssetTypes[asset.behavior] && shouldLoadIcuAsset(asset, preferredIcuAsset)) { expected_instantiated_assets_count++; } - if (!skipDownloadsByAssetTypes[asset.behavior]) { + if (!skipDownloadsByAssetTypes[asset.behavior] && shouldLoadIcuAsset(asset, preferredIcuAsset)) { expected_downloaded_assets_count++; promises_of_assets.push(start_asset_download(asset)); } @@ -101,10 +133,10 @@ export async function mono_download_assets(): Promise { const headersOnly = skipBufferByAssetTypes[asset.behavior]; if (!headersOnly) { mono_assert(asset.isOptional, "Expected asset to have the downloaded buffer"); - if (!skipDownloadsByAssetTypes[asset.behavior]) { + if (!skipDownloadsByAssetTypes[asset.behavior] && shouldLoadIcuAsset(asset, preferredIcuAsset)) { expected_downloaded_assets_count--; } - if (!skipInstantiateByAssetTypes[asset.behavior]) { + if (!skipInstantiateByAssetTypes[asset.behavior] && shouldLoadIcuAsset(asset, preferredIcuAsset)) { expected_instantiated_assets_count--; } } else { diff --git a/src/mono/wasm/testassets/icudt_custom.dat b/src/mono/wasm/testassets/icudt_custom.dat new file mode 100644 index 00000000000000..703d9704375e08 Binary files /dev/null and b/src/mono/wasm/testassets/icudt_custom.dat differ diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs index c99a05445407c0..89e3322cac2949 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilder.cs +++ b/src/tasks/WasmAppBuilder/WasmAppBuilder.cs @@ -125,8 +125,8 @@ protected override bool ValidateArguments() if (!File.Exists(MainJS)) throw new LogAsErrorException($"File MainJS='{MainJS}' doesn't exist."); - if (!InvariantGlobalization && string.IsNullOrEmpty(IcuDataFileName)) - throw new LogAsErrorException("IcuDataFileName property shouldn't be empty if InvariantGlobalization=false"); + if (!InvariantGlobalization && (IcuDataFileNames == null || IcuDataFileNames.Length == 0)) + throw new LogAsErrorException($"{nameof(IcuDataFileNames)} property shouldn't be empty when {nameof(InvariantGlobalization)}=false"); if (Assemblies.Length == 0) { @@ -308,7 +308,19 @@ protected override bool ExecuteInternal() } if (!InvariantGlobalization) - config.Assets.Add(new IcuData(IcuDataFileName!) { LoadRemote = RemoteSources?.Length > 0 }); + { + bool loadRemote = RemoteSources?.Length > 0; + foreach (var idfn in IcuDataFileNames) + { + if (!File.Exists(idfn)) + { + Log.LogError($"Expected the file defined as ICU resource: {idfn} to exist but it does not."); + return false; + } + config.Assets.Add(new IcuData(Path.GetFileName(idfn)) { LoadRemote = loadRemote }); + } + } + config.Assets.Add(new VfsEntry ("dotnet.timezones.blat") { VirtualPath = "/usr/share/zoneinfo/"}); config.Assets.Add(new WasmEntry ("dotnet.wasm") ); diff --git a/src/tasks/WasmAppBuilder/WasmAppBuilderBaseTask.cs b/src/tasks/WasmAppBuilder/WasmAppBuilderBaseTask.cs index 11d0f7fd9d6acb..c6c390f2bd2ba4 100644 --- a/src/tasks/WasmAppBuilder/WasmAppBuilderBaseTask.cs +++ b/src/tasks/WasmAppBuilder/WasmAppBuilderBaseTask.cs @@ -34,7 +34,7 @@ public abstract class WasmAppBuilderBaseTask : Task // full list of ICU data files we produce can be found here: // https://github.com/dotnet/icu/tree/maint/maint-67/icu-filters - public string? IcuDataFileName { get; set; } + public string[] IcuDataFileNames { get; set; } = Array.Empty(); public int DebugLevel { get; set; } public ITaskItem[] SatelliteAssemblies { get; set; } = Array.Empty(); diff --git a/src/tasks/WasmAppBuilder/wasi/WasiAppBuilder.cs b/src/tasks/WasmAppBuilder/wasi/WasiAppBuilder.cs index 0269e6721cf2a5..8ad5ee7960de9f 100644 --- a/src/tasks/WasmAppBuilder/wasi/WasiAppBuilder.cs +++ b/src/tasks/WasmAppBuilder/wasi/WasiAppBuilder.cs @@ -17,8 +17,8 @@ protected override bool ValidateArguments() if (!base.ValidateArguments()) return false; - if (!InvariantGlobalization && string.IsNullOrEmpty(IcuDataFileName)) - throw new LogAsErrorException("IcuDataFileName property shouldn't be empty if InvariantGlobalization=false"); + if (!InvariantGlobalization && (IcuDataFileNames == null || IcuDataFileNames.Length == 0)) + throw new LogAsErrorException($"{nameof(IcuDataFileNames)} property shouldn't be empty when {nameof(InvariantGlobalization)}=false"); if (Assemblies.Length == 0 && !IsSingleFileBundle) {