From db56d408e52911a24afa372125721c1d4190864c Mon Sep 17 00:00:00 2001 From: Elinor Fung Date: Mon, 18 Sep 2023 09:35:46 -0700 Subject: [PATCH 01/17] Check for .deps.json when enumerating framework paths (#92033) This adds a check for the framework's .deps.json instead of just the existence of the directory. To avoid extra file checks in the normal/happy path (where all framework version folder are valid), when resolving, it only does the check after resolving the best version match. And if that version directory isn't valid, it tries resolving again without it. Backport of #90035 --- .../FrameworkResolution/MultipleHives.cs | 20 ++++- .../HostActivation.Tests/NativeHostApis.cs | 14 +++- .../tests/TestUtils/DotNetBuilder.cs | 40 ++++++--- src/native/corehost/fxr/framework_info.cpp | 81 ++++++++++--------- src/native/corehost/fxr/framework_info.h | 2 +- src/native/corehost/fxr/fx_resolver.cpp | 51 +++++++----- .../corehost/fxr/fx_resolver.messages.cpp | 4 +- src/native/corehost/fxr/hostfxr.cpp | 2 +- 8 files changed, 138 insertions(+), 76 deletions(-) diff --git a/src/installer/tests/HostActivation.Tests/FrameworkResolution/MultipleHives.cs b/src/installer/tests/HostActivation.Tests/FrameworkResolution/MultipleHives.cs index 52236a4cc4076d..791a77b329c2ef 100644 --- a/src/installer/tests/HostActivation.Tests/FrameworkResolution/MultipleHives.cs +++ b/src/installer/tests/HostActivation.Tests/FrameworkResolution/MultipleHives.cs @@ -59,7 +59,8 @@ public void FrameworkHiveSelection(string requestedVersion, string tfm, bool? mu .WithTfm(tfm) .WithFramework(MicrosoftNETCoreApp, requestedVersion), multiLevelLookup) - .ShouldHaveResolvedFramework(MicrosoftNETCoreApp, resolvedVersion); + .ShouldHaveResolvedFramework(MicrosoftNETCoreApp, resolvedVersion) + .And.HaveStdErrContaining($"Ignoring FX version [{requestedVersion}] without .deps.json"); } [Fact] @@ -164,7 +165,8 @@ public void ListRuntimes(bool? multiLevelLookup) new TestSettings().WithCommandLine("--list-runtimes"), multiLevelLookup, testApp: null) - .Should().HaveStdOut(expectedOutput); + .Should().HaveStdOut(expectedOutput) + .And.HaveStdErrContaining("Ignoring FX version [9999.9.9] without .deps.json"); } [Theory] @@ -187,7 +189,8 @@ public void DotnetInfo(bool? multiLevelLookup) new TestSettings().WithCommandLine("--info"), multiLevelLookup, testApp: null) - .Should().HaveStdOutContaining(expectedOutput); + .Should().HaveStdOutContaining(expectedOutput) + .And.HaveStdErrContaining("Ignoring FX version [9999.9.9] without .deps.json"); } [Theory] @@ -217,7 +220,8 @@ public void FrameworkResolutionError(string tfm, bool? multiLevelLookup, bool ef multiLevelLookup) .Should().Fail() .And.HaveStdErrContaining(expectedOutput) - .And.HaveStdErrContaining("https://aka.ms/dotnet/app-launch-failed"); + .And.HaveStdErrContaining("https://aka.ms/dotnet/app-launch-failed") + .And.HaveStdErrContaining("Ignoring FX version [9999.9.9] without .deps.json"); } private CommandResult RunTest(Func runtimeConfig, bool? multiLevelLookup = true) @@ -261,6 +265,14 @@ public SharedTestState() .AddMicrosoftNETCoreAppFrameworkMockHostPolicy("7.1.2") .Build(); + // Empty Microsoft.NETCore.App directory - should not be recognized as a valid framework + // Version is the best match for some test cases, but they should be ignored + string netCoreAppDir = Path.Combine(DotNetMainHive.BinPath, "shared", Constants.MicrosoftNETCoreApp); + Directory.CreateDirectory(Path.Combine(netCoreAppDir, "5.0.0")); + Directory.CreateDirectory(Path.Combine(netCoreAppDir, "6.0.0")); + Directory.CreateDirectory(Path.Combine(netCoreAppDir, "7.0.0")); + Directory.CreateDirectory(Path.Combine(netCoreAppDir, "9999.9.9")); + DotNetGlobalHive = DotNet("GlobalHive") .AddMicrosoftNETCoreAppFrameworkMockHostPolicy("5.1.2") .AddMicrosoftNETCoreAppFrameworkMockHostPolicy("6.1.4") diff --git a/src/installer/tests/HostActivation.Tests/NativeHostApis.cs b/src/installer/tests/HostActivation.Tests/NativeHostApis.cs index 213b3790335ce4..47a839fc4e0e2f 100644 --- a/src/installer/tests/HostActivation.Tests/NativeHostApis.cs +++ b/src/installer/tests/HostActivation.Tests/NativeHostApis.cs @@ -114,12 +114,22 @@ public SdkResolutionFixture(SharedTestState state) foreach ((string fwName, string[] fwVersions) in ProgramFilesGlobalFrameworks) { foreach (string fwVersion in fwVersions) - Directory.CreateDirectory(Path.Combine(ProgramFilesGlobalFrameworksDir, fwName, fwVersion)); + AddFrameworkDirectory(ProgramFilesGlobalFrameworksDir, fwName, fwVersion); } foreach ((string fwName, string[] fwVersions) in LocalFrameworks) { foreach (string fwVersion in fwVersions) - Directory.CreateDirectory(Path.Combine(LocalFrameworksDir, fwName, fwVersion)); + AddFrameworkDirectory(LocalFrameworksDir, fwName, fwVersion); + + // Empty framework directory - this should not be recognized as a valid framework directory + Directory.CreateDirectory(Path.Combine(LocalFrameworksDir, fwName, "9.9.9")); + } + + static void AddFrameworkDirectory(string frameworkDir, string name, string version) + { + string versionDir = Path.Combine(frameworkDir, name, version); + Directory.CreateDirectory(versionDir); + File.WriteAllText(Path.Combine(versionDir, $"{name}.deps.json"), string.Empty); } } } diff --git a/src/installer/tests/TestUtils/DotNetBuilder.cs b/src/installer/tests/TestUtils/DotNetBuilder.cs index 53a61568d70747..fcfe1cd7cbb1d5 100644 --- a/src/installer/tests/TestUtils/DotNetBuilder.cs +++ b/src/installer/tests/TestUtils/DotNetBuilder.cs @@ -51,8 +51,7 @@ public DotNetBuilder(string basePath, string builtDotnet, string name) public DotNetBuilder AddMicrosoftNETCoreAppFrameworkMockHostPolicy(string version) { // ./shared/Microsoft.NETCore.App/ - create a mock of the root framework - string netCoreAppPath = Path.Combine(_path, "shared", "Microsoft.NETCore.App", version); - Directory.CreateDirectory(netCoreAppPath); + string netCoreAppPath = AddFramework(Constants.MicrosoftNETCoreApp, version); // ./shared/Microsoft.NETCore.App//hostpolicy.dll - this is a mock, will not actually load CoreCLR string mockHostPolicyFileName = RuntimeInformationExtensions.GetSharedLibraryFileNameForCurrentPlatform("mockhostpolicy"); @@ -122,8 +121,7 @@ public DotNetBuilder RemoveHostFxr(Version version = null) public DotNetBuilder AddMicrosoftNETCoreAppFrameworkMockCoreClr(string version, Action customizer = null) { // ./shared/Microsoft.NETCore.App/ - create a mock of the root framework - string netCoreAppPath = Path.Combine(_path, "shared", "Microsoft.NETCore.App", version); - Directory.CreateDirectory(netCoreAppPath); + string netCoreAppPath = AddFramework(Constants.MicrosoftNETCoreApp, version); string hostPolicyFileName = RuntimeInformationExtensions.GetSharedLibraryFileNameForCurrentPlatform("hostpolicy"); string coreclrFileName = RuntimeInformationExtensions.GetSharedLibraryFileNameForCurrentPlatform("coreclr"); @@ -131,9 +129,9 @@ public DotNetBuilder AddMicrosoftNETCoreAppFrameworkMockCoreClr(string version, string currentRid = _repoDirectories.TargetRID; - NetCoreAppBuilder.ForNETCoreApp("Microsoft.NETCore.App", currentRid) + NetCoreAppBuilder.ForNETCoreApp(Constants.MicrosoftNETCoreApp, currentRid) .WithStandardRuntimeFallbacks() - .WithProject("Microsoft.NETCore.App", version, p => p + .WithProject(Constants.MicrosoftNETCoreApp, version, p => p .WithNativeLibraryGroup(null, g => g // ./shared/Microsoft.NETCore.App//coreclr.dll - this is a mock, will not actually run CoreClr .WithAsset((new NetCoreAppBuilder.RuntimeFileBuilder($"runtimes/{currentRid}/native/{coreclrFileName}")) @@ -158,7 +156,7 @@ public DotNetBuilder AddMicrosoftNETCoreAppFrameworkMockCoreClr(string version, /// Framework version /// Customization function for the runtime config /// - /// The added mock framework will only contain a runtime.config.json file. + /// The added mock framework will only contain a deps.json and a runtime.config.json file. /// public DotNetBuilder AddFramework( string name, @@ -166,12 +164,11 @@ public DotNetBuilder AddFramework( Action runtimeConfigCustomizer, Action frameworkCustomizer = null) { - // ./shared// - create a mock of effectively empty non-root framework - string path = Path.Combine(_path, "shared", name, version); - Directory.CreateDirectory(path); + // ./shared// - create a mock of the framework + string path = AddFramework(name, version); // ./shared///.runtimeconfig.json - runtime config which can be customized - RuntimeConfig runtimeConfig = new RuntimeConfig(Path.Combine(path, name + ".runtimeconfig.json")); + RuntimeConfig runtimeConfig = new RuntimeConfig(Path.Combine(path, $"{name}.runtimeconfig.json")); runtimeConfigCustomizer(runtimeConfig); runtimeConfig.Save(); @@ -197,6 +194,27 @@ public DotNetBuilder AddMockSDK( return this; } + /// + /// Add a minimal mock framework with the specified framework name and version + /// + /// Framework name + /// Framework version + /// Framework directory + /// + /// The added mock framework will only contain a deps.json. + /// + private string AddFramework(string name, string version) + { + // ./shared// - create a mock of effectively the framework + string path = Path.Combine(_path, "shared", name, version); + Directory.CreateDirectory(path); + + // ./shared///.deps.json - empty file + File.WriteAllText(Path.Combine(path, $"{name}.deps.json"), string.Empty); + + return path; + } + public DotNetCli Build() { return new DotNetCli(_path); diff --git a/src/native/corehost/fxr/framework_info.cpp b/src/native/corehost/fxr/framework_info.cpp index b8bb45e3278b8b..0e2213c562c7cc 100644 --- a/src/native/corehost/fxr/framework_info.cpp +++ b/src/native/corehost/fxr/framework_info.cpp @@ -34,7 +34,7 @@ bool compare_by_name_and_version(const framework_info &a, const framework_info & /*static*/ void framework_info::get_all_framework_infos( const pal::string_t& own_dir, - const pal::string_t& fx_name, + const pal::char_t* fx_name, bool disable_multilevel_lookup, std::vector* framework_infos) { @@ -43,49 +43,58 @@ bool compare_by_name_and_version(const framework_info &a, const framework_info & int32_t hive_depth = 0; - for (pal::string_t dir : hive_dir) + for (const pal::string_t& dir : hive_dir) { auto fx_shared_dir = dir; append_path(&fx_shared_dir, _X("shared")); - if (pal::directory_exists(fx_shared_dir)) + if (!pal::directory_exists(fx_shared_dir)) + continue; + + std::vector fx_names; + if (fx_name != nullptr) { - std::vector fx_names; - if (fx_name.length()) - { - // Use the provided framework name - fx_names.push_back(fx_name); - } - else - { - // Read all frameworks, including "Microsoft.NETCore.App" - pal::readdir_onlydirectories(fx_shared_dir, &fx_names); - } + // Use the provided framework name + fx_names.push_back(fx_name); + } + else + { + // Read all frameworks, including "Microsoft.NETCore.App" + pal::readdir_onlydirectories(fx_shared_dir, &fx_names); + } - for (pal::string_t fx_name_local : fx_names) - { - auto fx_dir = fx_shared_dir; - append_path(&fx_dir, fx_name_local.c_str()); + for (const pal::string_t& fx_name_local : fx_names) + { + auto fx_dir = fx_shared_dir; + append_path(&fx_dir, fx_name_local.c_str()); + + if (!pal::directory_exists(fx_dir)) + continue; + + trace::verbose(_X("Gathering FX locations in [%s]"), fx_dir.c_str()); - if (pal::directory_exists(fx_dir)) + std::vector versions; + pal::readdir_onlydirectories(fx_dir, &versions); + for (const pal::string_t& ver : versions) + { + // Make sure we filter out any non-version folders. + fx_ver_t parsed; + if (!fx_ver_t::parse(ver, &parsed, false)) + continue; + + // Check that the framework's .deps.json exists. + pal::string_t fx_version_dir = fx_dir; + append_path(&fx_version_dir, ver.c_str()); + if (!library_exists_in_dir(fx_version_dir, fx_name_local + _X(".deps.json"), nullptr)) { - trace::verbose(_X("Gathering FX locations in [%s]"), fx_dir.c_str()); - - std::vector versions; - pal::readdir_onlydirectories(fx_dir, &versions); - for (const auto& ver : versions) - { - // Make sure we filter out any non-version folders. - fx_ver_t parsed; - if (fx_ver_t::parse(ver, &parsed, false)) - { - trace::verbose(_X("Found FX version [%s]"), ver.c_str()); - - framework_info info(fx_name_local, fx_dir, parsed, hive_depth); - framework_infos->push_back(info); - } - } + trace::verbose(_X("Ignoring FX version [%s] without .deps.json"), ver.c_str()); + continue; } + + trace::verbose(_X("Found FX version [%s]"), ver.c_str()); + + framework_info info(fx_name_local, fx_dir, parsed, hive_depth); + framework_infos->push_back(info); } } @@ -98,7 +107,7 @@ bool compare_by_name_and_version(const framework_info &a, const framework_info & /*static*/ bool framework_info::print_all_frameworks(const pal::string_t& own_dir, const pal::string_t& leading_whitespace) { std::vector framework_infos; - get_all_framework_infos(own_dir, _X(""), /*disable_multilevel_lookup*/ true, &framework_infos); + get_all_framework_infos(own_dir, nullptr, /*disable_multilevel_lookup*/ true, &framework_infos); for (framework_info info : framework_infos) { trace::println(_X("%s%s %s [%s]"), leading_whitespace.c_str(), info.name.c_str(), info.version.as_str().c_str(), info.path.c_str()); diff --git a/src/native/corehost/fxr/framework_info.h b/src/native/corehost/fxr/framework_info.h index 8e95bdea0cc61a..926f58cf0d7b06 100644 --- a/src/native/corehost/fxr/framework_info.h +++ b/src/native/corehost/fxr/framework_info.h @@ -17,7 +17,7 @@ struct framework_info static void get_all_framework_infos( const pal::string_t& own_dir, - const pal::string_t& fx_name, + const pal::char_t* fx_name, bool disable_multilevel_lookup, std::vector* framework_infos); diff --git a/src/native/corehost/fxr/fx_resolver.cpp b/src/native/corehost/fxr/fx_resolver.cpp index faa40ed9228a90..7bb65d3a374b24 100644 --- a/src/native/corehost/fxr/fx_resolver.cpp +++ b/src/native/corehost/fxr/fx_resolver.cpp @@ -174,9 +174,6 @@ namespace if (best_match == fx_ver_t()) { - // This is not strictly necessary, we just need to return version which doesn't exist. - // But it's cleaner to return the desider reference then invalid -1.-1.-1 version. - best_match = fx_ref.get_fx_version_number(); trace::verbose(_X("Framework reference didn't resolve to any available version.")); } else if (trace::is_enabled()) @@ -212,7 +209,8 @@ namespace pal::string_t selected_fx_version; fx_ver_t selected_ver; - for (pal::string_t dir : hive_dir) + pal::string_t deps_file_name = fx_ref.get_fx_name() + _X(".deps.json"); + for (pal::string_t& dir : hive_dir) { auto fx_dir = dir; trace::verbose(_X("Searching FX directory in [%s]"), fx_dir.c_str()); @@ -236,7 +234,7 @@ namespace fx_ref.get_fx_version().c_str()); append_path(&fx_dir, fx_ref.get_fx_version().c_str()); - if (pal::directory_exists(fx_dir)) + if (library_exists_in_dir(fx_dir, deps_file_name, nullptr)) { selected_fx_dir = fx_dir; selected_fx_version = fx_ref.get_fx_version(); @@ -259,24 +257,39 @@ namespace } fx_ver_t resolved_ver = resolve_framework_reference_from_version_list(version_list, fx_ref); - - pal::string_t resolved_ver_str = resolved_ver.as_str(); - append_path(&fx_dir, resolved_ver_str.c_str()); - - if (pal::directory_exists(fx_dir)) + while (resolved_ver != fx_ver_t()) { - if (selected_ver != fx_ver_t()) + pal::string_t resolved_ver_str = resolved_ver.as_str(); + pal::string_t resolved_fx_dir = fx_dir; + append_path(&resolved_fx_dir, resolved_ver_str.c_str()); + + // Check that the framework's .deps.json exists. To minimize the file checks done in the most common + // scenario (.deps.json exists), only check after resolving the version and if the .deps.json doesn't + // exist, attempt resolving again without that version. + if (!library_exists_in_dir(resolved_fx_dir, deps_file_name, nullptr)) { - // Compare the previous hive_dir selection with the current hive_dir to see which one is the better match - resolved_ver = resolve_framework_reference_from_version_list({ resolved_ver, selected_ver }, fx_ref); + // Remove the version and try resolving again + trace::verbose(_X("Ignoring FX version [%s] without .deps.json"), resolved_ver_str.c_str()); + version_list.erase(std::find(version_list.cbegin(), version_list.cend(), resolved_ver)); + resolved_ver = resolve_framework_reference_from_version_list(version_list, fx_ref); } - - if (resolved_ver != selected_ver) + else { - trace::verbose(_X("Changing Selected FX version from [%s] to [%s]"), selected_fx_dir.c_str(), fx_dir.c_str()); - selected_ver = resolved_ver; - selected_fx_dir = fx_dir; - selected_fx_version = resolved_ver_str; + if (selected_ver != fx_ver_t()) + { + // Compare the previous hive_dir selection with the current hive_dir to see which one is the better match + resolved_ver = resolve_framework_reference_from_version_list({ resolved_ver, selected_ver }, fx_ref); + } + + if (resolved_ver != selected_ver) + { + trace::verbose(_X("Changing Selected FX version from [%s] to [%s]"), selected_fx_dir.c_str(), resolved_fx_dir.c_str()); + selected_ver = resolved_ver; + selected_fx_dir = resolved_fx_dir; + selected_fx_version = resolved_ver_str; + } + + break; } } } diff --git a/src/native/corehost/fxr/fx_resolver.messages.cpp b/src/native/corehost/fxr/fx_resolver.messages.cpp index 1113e9ced98223..105618e13547c0 100644 --- a/src/native/corehost/fxr/fx_resolver.messages.cpp +++ b/src/native/corehost/fxr/fx_resolver.messages.cpp @@ -101,14 +101,14 @@ void fx_resolver_t::display_missing_framework_error( if (fx_dir.length()) { fx_ver_dirs = fx_dir; - framework_info::get_all_framework_infos(get_directory(fx_dir), fx_name, disable_multilevel_lookup, &framework_infos); + framework_info::get_all_framework_infos(get_directory(fx_dir), fx_name.c_str(), disable_multilevel_lookup, &framework_infos); } else { fx_ver_dirs = dotnet_root; } - framework_info::get_all_framework_infos(dotnet_root, fx_name, disable_multilevel_lookup, &framework_infos); + framework_info::get_all_framework_infos(dotnet_root, fx_name.c_str(), disable_multilevel_lookup, &framework_infos); // Display the error message about missing FX. if (fx_version.length()) diff --git a/src/native/corehost/fxr/hostfxr.cpp b/src/native/corehost/fxr/hostfxr.cpp index 96f8b0476117a3..82b29f9103058d 100644 --- a/src/native/corehost/fxr/hostfxr.cpp +++ b/src/native/corehost/fxr/hostfxr.cpp @@ -438,7 +438,7 @@ SHARED_API int32_t HOSTFXR_CALLTYPE hostfxr_get_dotnet_environment_info( } std::vector framework_infos; - framework_info::get_all_framework_infos(dotnet_dir, _X(""), /*disable_multilevel_lookup*/ true, &framework_infos); + framework_info::get_all_framework_infos(dotnet_dir, nullptr, /*disable_multilevel_lookup*/ true, &framework_infos); std::vector environment_framework_infos; std::vector framework_versions; From 46a293b7c6d7703c9800e136f2bbba57c608d9d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20S=C3=A1nchez=20L=C3=B3pez?= <1175054+carlossanlop@users.noreply.github.com> Date: Thu, 28 Sep 2023 19:17:48 -0600 Subject: [PATCH 02/17] [release/7.0] Update Ubuntu 16.04 amd64 queues to 22.04 (#92796) --- eng/pipelines/coreclr/templates/helix-queues-setup.yml | 2 +- eng/pipelines/libraries/helix-queues-setup.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/coreclr/templates/helix-queues-setup.yml b/eng/pipelines/coreclr/templates/helix-queues-setup.yml index 59475dec6726bc..f61ee6d3efe4a4 100644 --- a/eng/pipelines/coreclr/templates/helix-queues-setup.yml +++ b/eng/pipelines/coreclr/templates/helix-queues-setup.yml @@ -67,7 +67,7 @@ jobs: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - (Alpine.314.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.14-helix-amd64 - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - (Alpine.314.Amd64)ubuntu.1604.amd64@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.14-helix-amd64 + - (Alpine.314.Amd64)ubuntu.2204.amd64.svc@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.14-helix-amd64 # Linux musl arm32 - ${{ if eq(parameters.platform, 'Linux_musl_arm') }}: diff --git a/eng/pipelines/libraries/helix-queues-setup.yml b/eng/pipelines/libraries/helix-queues-setup.yml index 54a2d7280986f4..554d44135ba3be 100644 --- a/eng/pipelines/libraries/helix-queues-setup.yml +++ b/eng/pipelines/libraries/helix-queues-setup.yml @@ -71,10 +71,10 @@ jobs: - (Fedora.34.Amd64.Open)ubuntu.1804.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-34-helix - Ubuntu.2204.Amd64.Open - (Debian.11.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-amd64 - - (Mariner.1.0.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-1.0-helix - - (openSUSE.15.2.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:opensuse-15.2-helix-amd64 + - (Mariner.1.0.Amd64.Open)ubuntu.2204.amd64.open.svc@mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-1.0-helix + - (openSUSE.15.2.Amd64.Open)ubuntu.2204.amd64.open.svc@mcr.microsoft.com/dotnet-buildtools/prereqs:opensuse-15.2-helix-amd64 - ${{ if or(ne(parameters.jobParameters.isExtraPlatforms, true), eq(parameters.jobParameters.includeAllPlatforms, true)) }}: - - (Centos.7.Amd64.Open)Ubuntu.1604.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-mlnet-helix + - (Centos.7.Amd64.Open)Ubuntu.2204.Amd64.Open.svc@mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-mlnet-helix - RedHat.7.Amd64.Open - (Debian.10.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-amd64 - Ubuntu.1804.Amd64.Open From c7425f7f228c7bb86761eaeda601440c5769e458 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 29 Sep 2023 19:16:17 -0700 Subject: [PATCH 03/17] [release/7.0-staging] Fix JsonDocument thread safety. (#92831) * Fix JsonDocument thread safety. Co-authored-by: stoub@microsoft.com * Update ServicingVersion --------- Co-authored-by: Eirik Tsarpalis --- .../src/System.Text.Json.csproj | 4 +- .../System/Text/Json/Document/JsonDocument.cs | 32 ++------------ .../JsonDocumentTests.cs | 42 +++++++++++++++++-- 3 files changed, 43 insertions(+), 35 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System.Text.Json.csproj b/src/libraries/System.Text.Json/src/System.Text.Json.csproj index 4f20eb177dcc3c..7f1ee403e50672 100644 --- a/src/libraries/System.Text.Json/src/System.Text.Json.csproj +++ b/src/libraries/System.Text.Json/src/System.Text.Json.csproj @@ -8,8 +8,8 @@ CS8969 true true - false - 3 + true + 4 true Provides high-performance and low-allocating types that serialize objects to JavaScript Object Notation (JSON) text and deserialize JSON text to objects, with UTF-8 support built-in. Also provides types to read and write JSON text encoded as UTF-8, and to create an in-memory document object model (DOM), that is read-only, for random access of the JSON elements within a structured view of the data. diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonDocument.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonDocument.cs index 2c15d1e63860a2..45fa176cd2e4db 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonDocument.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonDocument.cs @@ -27,8 +27,6 @@ public sealed partial class JsonDocument : IDisposable private byte[]? _extraRentedArrayPoolBytes; private PooledByteBufferWriter? _extraPooledByteBufferWriter; - private (int, string?) _lastIndexAndString = (-1, null); - internal bool IsDisposable { get; } /// @@ -266,14 +264,6 @@ private ReadOnlyMemory GetPropertyRawValue(int valueIndex) { CheckNotDisposed(); - (int lastIdx, string? lastString) = _lastIndexAndString; - - if (lastIdx == index) - { - Debug.Assert(lastString != null); - return lastString; - } - DbRow row = _parsedData.Get(index); JsonTokenType tokenType = row.TokenType; @@ -288,18 +278,9 @@ private ReadOnlyMemory GetPropertyRawValue(int valueIndex) ReadOnlySpan data = _utf8Json.Span; ReadOnlySpan segment = data.Slice(row.Location, row.SizeOrLength); - if (row.HasComplexChildren) - { - lastString = JsonReaderHelper.GetUnescapedString(segment); - } - else - { - lastString = JsonReaderHelper.TranscodeHelper(segment); - } - - Debug.Assert(lastString != null); - _lastIndexAndString = (index, lastString); - return lastString; + return row.HasComplexChildren + ? JsonReaderHelper.GetUnescapedString(segment) + : JsonReaderHelper.TranscodeHelper(segment); } internal bool TextEquals(int index, ReadOnlySpan otherText, bool isPropertyName) @@ -308,13 +289,6 @@ internal bool TextEquals(int index, ReadOnlySpan otherText, bool isPropert int matchIndex = isPropertyName ? index - DbRow.Size : index; - (int lastIdx, string? lastString) = _lastIndexAndString; - - if (lastIdx == matchIndex) - { - return otherText.SequenceEqual(lastString.AsSpan()); - } - byte[]? otherUtf8TextArray = null; int length = checked(otherText.Length * JsonConstants.MaxExpansionFactorWhileTranscoding); diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonDocumentTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonDocumentTests.cs index cc3d827ca729fa..9bae2532b760c8 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonDocumentTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonDocumentTests.cs @@ -14,6 +14,7 @@ using System.Linq; using System.Runtime.InteropServices; using System.Threading.Tasks; +using System.Threading; namespace System.Text.Json.Tests { @@ -2744,11 +2745,9 @@ public static void ObjectEnumeratorIndependentWalk() Assert.Equal(test, property.Value.GetInt32()); test++; - // Subsequent read of the same JsonProperty doesn't allocate a new string - // (if another property is inspected from the same document that guarantee - // doesn't hold). + // Subsequent read of the same JsonProperty should return an equal string string propertyName2 = property.Name; - Assert.Same(propertyName, propertyName2); + Assert.Equal(propertyName, propertyName2); } test = 0; @@ -3607,6 +3606,41 @@ public static void NameEquals_Empty_Throws() } } + [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsThreadingSupported))] + [OuterLoop] // thread-safety / stress test + public static async Task GetString_ConcurrentUse_ThreadSafe() + { + using (JsonDocument doc = JsonDocument.Parse(SR.SimpleObjectJson)) + { + JsonElement first = doc.RootElement.GetProperty("first"); + JsonElement last = doc.RootElement.GetProperty("last"); + + const int Iters = 10_000; + using (var gate = new Barrier(2)) + { + await Task.WhenAll( + Task.Factory.StartNew(() => + { + gate.SignalAndWait(); + for (int i = 0; i < Iters; i++) + { + Assert.Equal("John", first.GetString()); + Assert.True(first.ValueEquals("John")); + } + }, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default), + Task.Factory.StartNew(() => + { + gate.SignalAndWait(); + for (int i = 0; i < Iters; i++) + { + Assert.Equal("Smith", last.GetString()); + Assert.True(last.ValueEquals("Smith")); + } + }, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default)); + } + } + } + private static void BuildSegmentedReader( out Utf8JsonReader reader, ReadOnlyMemory data, From 89d99c96bf4f1836d8aac907774aa95abd0d611f Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Tue, 3 Oct 2023 21:28:20 -0400 Subject: [PATCH 04/17] CI: runtime-wasm-perf: disable for PRs (#92978) --- eng/pipelines/runtime-wasm-perf.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/runtime-wasm-perf.yml b/eng/pipelines/runtime-wasm-perf.yml index 0ee6c1cdb9f890..0e49214ad09191 100644 --- a/eng/pipelines/runtime-wasm-perf.yml +++ b/eng/pipelines/runtime-wasm-perf.yml @@ -3,6 +3,7 @@ # UI to this, and thus avoid any scheduled triggers trigger: none +pr: none variables: - template: /eng/pipelines/common/variables.yml From 83ec4e3641b46bedb65f294d343335592e9d5011 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 4 Oct 2023 09:54:34 -0700 Subject: [PATCH 05/17] Update dependencies from https://github.com/dotnet/emsdk build 20231004.1 (#92991) Microsoft.NET.Workload.Emscripten.net6.Manifest-7.0.100 , Microsoft.NET.Workload.Emscripten.net7.Manifest-7.0.100 From Version 7.0.12 -> To Version 7.0.13 Co-authored-by: dotnet-maestro[bot] --- NuGet.config | 2 +- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/NuGet.config b/NuGet.config index 9c77e9437dd4b1..cc344fd0bdf393 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,7 +9,7 @@ - + From 7ee35b9e89e43ee0f6d08ff42a342c06e80d8180 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 6 Oct 2023 15:18:27 -0400 Subject: [PATCH 06/17] [release/7.0] [Mono] Race in init_method when using LLVM AOT. (#93006) Backport of #75088 to release/7.0-staging Fixes #81211 ## Customer Impact Customers targeting Apple platforms using LLVM AOT codegen (the default) in highly concurrent settings (such as firing off multiple simultaneous async HTTP requests) may experience unexpected behavior such as InvalidCastExceptions, NullReferenceExceptions or crashes. ## Testing Manual testing ## Risk Low. This code has been running on .NET 8 `main` for over a year in CI, as well as on some other non-mobile platforms --- * [Mono] Race in init_method when using LLVM AOT. When using LLVM AOT codegen, init_method updates two GOT slots. These slots are initialized as part of init_method, but there is a race between initialization of the two slots. Current implementation can have two threads running init_method for the same method, but as soon as: [got_slots [pindex]] = addr store is visible, it will trigger other threads to return back from init_method, but since that could happen before the corresponding LLVM AOT const slot is set, second thread will return to method calling init_method, load the LLVM aot const, and crash when trying to use it (since its still NULL). This crash is very rare but have been identified on x86/x64 CPU's, when one thread is either preempted between updating regular GOT slot and LLVM GOT slot or store into LLVM GOT slot gets delayed in store buffer. I have also been able to emulate the scenario in debugger, triggering the issue and crashing in the method loading from LLVM aot const slot. Fix change order of updates and make sure the update of LLVM aot const slot happens before memory_barrier, since: got [got_slots [pindex]] = addr; have release semantics in relation to addr and update of LLVM aot const slot. Fix also add acquire/release semantics for ji->type in init_method since it is used to guard if a thread ignores a patch or not and it should not be re-ordered with previous stores, since it can cause similar race conditions with updated slots. * Move register_jump_target_got_slot above mono_memory_barrier. * revert unintentional branding change --------- Co-authored-by: vseanreesermsft <78103370+vseanreesermsft@users.noreply.github.com> Co-authored-by: lateralusX Co-authored-by: Aleksey Kliger --- src/mono/mono/mini/aot-runtime.c | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/mono/mono/mini/aot-runtime.c b/src/mono/mono/mini/aot-runtime.c index 50639c2c397af2..36558cbdae7714 100644 --- a/src/mono/mono/mini/aot-runtime.c +++ b/src/mono/mono/mini/aot-runtime.c @@ -4633,10 +4633,13 @@ init_method (MonoAotModule *amodule, gpointer info, guint32 method_index, MonoMe * been initialized by load_method () for a static cctor before the cctor has * finished executing (#23242). */ - if (ji->type == MONO_PATCH_INFO_NONE) { - } else if (!got [got_slots [pindex]] || ji->type == MONO_PATCH_INFO_SFLDA) { + MonoJumpInfoType ji_type = ji->type; + LOAD_ACQUIRE_FENCE; + + if (ji_type == MONO_PATCH_INFO_NONE) { + } else if (!got [got_slots [pindex]] || ji_type == MONO_PATCH_INFO_SFLDA) { /* In llvm-only made, we might encounter shared methods */ - if (mono_llvm_only && ji->type == MONO_PATCH_INFO_METHOD && mono_method_check_context_used (ji->data.method)) { + if (mono_llvm_only && ji_type == MONO_PATCH_INFO_METHOD && mono_method_check_context_used (ji->data.method)) { g_assert (context); ji->data.method = mono_class_inflate_generic_method_checked (ji->data.method, context, error); if (!is_ok (error)) { @@ -4646,10 +4649,10 @@ init_method (MonoAotModule *amodule, gpointer info, guint32 method_index, MonoMe } } /* This cannot be resolved in mono_resolve_patch_target () */ - if (ji->type == MONO_PATCH_INFO_AOT_JIT_INFO) { + if (ji_type == MONO_PATCH_INFO_AOT_JIT_INFO) { // FIXME: Lookup using the index jinfo = mono_aot_find_jit_info (amodule->assembly->image, code); - ji->type = MONO_PATCH_INFO_ABS; + ji->type = ji_type = MONO_PATCH_INFO_ABS; ji->data.target = jinfo; } addr = mono_resolve_patch_target (method, code, ji, TRUE, error); @@ -4658,18 +4661,19 @@ init_method (MonoAotModule *amodule, gpointer info, guint32 method_index, MonoMe mono_mempool_destroy (mp); return FALSE; } - if (ji->type == MONO_PATCH_INFO_METHOD_JUMP) + if (ji_type == MONO_PATCH_INFO_METHOD_JUMP) { addr = mono_create_ftnptr (addr); - mono_memory_barrier (); - got [got_slots [pindex]] = addr; - if (ji->type == MONO_PATCH_INFO_METHOD_JUMP) register_jump_target_got_slot (ji->data.method, &(got [got_slots [pindex]])); - + } if (llvm) { void (*init_aotconst) (int, gpointer) = (void (*)(int, gpointer))amodule->info.llvm_init_aotconst; init_aotconst (got_slots [pindex], addr); } + mono_memory_barrier (); + got [got_slots [pindex]] = addr; } + + STORE_RELEASE_FENCE; ji->type = MONO_PATCH_INFO_NONE; } From be8f7447f6cb3d0d64f3397094cb59fda0b594f8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 9 Oct 2023 12:56:07 -0700 Subject: [PATCH 07/17] [release/7.0-staging] Fix a memory leak in runtime interop stubs when using an array of structs of types that use old-style managed marshalers (#93148) Co-authored-by: Jeremy Koritzinsky --- src/coreclr/vm/ilmarshalers.h | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/src/coreclr/vm/ilmarshalers.h b/src/coreclr/vm/ilmarshalers.h index f3c9f31628f156..61ff10ac2b2b86 100644 --- a/src/coreclr/vm/ilmarshalers.h +++ b/src/coreclr/vm/ilmarshalers.h @@ -3138,39 +3138,13 @@ class ILMngdMarshaler : public ILMarshaler void EmitClearNative(ILCodeStream* pslILEmit) override { WRAPPER_NO_CONTRACT; - ILCodeLabel* pNoManagedValueLabel = nullptr; - if (IsFieldMarshal(m_dwMarshalFlags)) - { - pNoManagedValueLabel = pslILEmit->NewCodeLabel(); - pslILEmit->EmitLDARG(StructMarshalStubs::MANAGED_STRUCT_ARGIDX); - pslILEmit->EmitBRFALSE(pNoManagedValueLabel); - } - EmitCallMngdMarshalerMethod(pslILEmit, GetClearNativeMethod()); - - if (IsFieldMarshal(m_dwMarshalFlags)) - { - pslILEmit->EmitLabel(pNoManagedValueLabel); - } } void EmitClearNativeContents(ILCodeStream* pslILEmit) override { WRAPPER_NO_CONTRACT; - ILCodeLabel* pNoManagedValueLabel = nullptr; - if (IsFieldMarshal(m_dwMarshalFlags)) - { - pNoManagedValueLabel = pslILEmit->NewCodeLabel(); - pslILEmit->EmitLDARG(StructMarshalStubs::MANAGED_STRUCT_ARGIDX); - pslILEmit->EmitBRFALSE(pNoManagedValueLabel); - } - EmitCallMngdMarshalerMethod(pslILEmit, GetClearNativeContentsMethod()); - - if (IsFieldMarshal(m_dwMarshalFlags)) - { - pslILEmit->EmitLabel(pNoManagedValueLabel); - } } bool NeedsClearCLR() override From 341b81cc80dab09cb93632dbdfe8a41283ccb5cf Mon Sep 17 00:00:00 2001 From: Koundinya Veluri Date: Wed, 11 Oct 2023 15:31:58 -0700 Subject: [PATCH 08/17] Make thread pool thread timeouts configurable (#92987) - Added two config options, one that configures the worker and wait thread timeouts, and another that enables keeping some number of worker threads alive after they are created - This enables services that take periodic traffic to keep some worker threads around for better latency, while allowing extra threads to time out as appropriate for the service --- src/coreclr/inc/clrconfigvalues.h | 2 + src/coreclr/vm/comthreadpool.cpp | 28 ++++++ src/coreclr/vm/eeconfig.cpp | 16 ++++ src/coreclr/vm/eeconfig.h | 6 ++ src/coreclr/vm/win32threadpool.cpp | 85 +++++++++++++++++-- src/coreclr/vm/win32threadpool.h | 7 +- .../PortableThreadPool.WorkerThread.cs | 46 +++++++++- .../System/Threading/PortableThreadPool.cs | 17 +++- 8 files changed, 197 insertions(+), 10 deletions(-) diff --git a/src/coreclr/inc/clrconfigvalues.h b/src/coreclr/inc/clrconfigvalues.h index 5c4098dd07b5b0..4668c3dd49ea95 100644 --- a/src/coreclr/inc/clrconfigvalues.h +++ b/src/coreclr/inc/clrconfigvalues.h @@ -542,6 +542,8 @@ RETAIL_CONFIG_DWORD_INFO(INTERNAL_ThreadPool_UnfairSemaphoreSpinLimit, W("Thread #else // !TARGET_ARM64 RETAIL_CONFIG_DWORD_INFO(INTERNAL_ThreadPool_UnfairSemaphoreSpinLimit, W("ThreadPool_UnfairSemaphoreSpinLimit"), 0x46, "Maximum number of spins a thread pool worker thread performs before waiting for work") #endif // TARGET_ARM64 +RETAIL_CONFIG_DWORD_INFO_EX(EXTERNAL_ThreadPool_ThreadTimeoutMs, W("ThreadPool_ThreadTimeoutMs"), (DWORD)-2, "The amount of time in milliseconds a thread pool thread waits without having done any work before timing out and exiting. Set to -1 to disable the timeout. Applies to worker threads, completion port threads, and wait threads. Also see the ThreadPool_ThreadsToKeepAlive config value for relevant information.", CLRConfig::LookupOptions::ParseIntegerAsBase10) +RETAIL_CONFIG_DWORD_INFO_EX(EXTERNAL_ThreadPool_ThreadsToKeepAlive, W("ThreadPool_ThreadsToKeepAlive"), 0, "The number of worker or completion port threads to keep alive after they are created. Set to -1 to keep all created worker or completion port threads alive. When the ThreadPool_ThreadTimeoutMs config value is also set, for worker and completion port threads the timeout applies to threads in the respective pool that are in excess of the number configured for ThreadPool_ThreadsToKeepAlive.", CLRConfig::LookupOptions::ParseIntegerAsBase10) RETAIL_CONFIG_DWORD_INFO(INTERNAL_HillClimbing_Disable, W("HillClimbing_Disable"), 0, "Disables hill climbing for thread adjustments in the thread pool"); RETAIL_CONFIG_DWORD_INFO(INTERNAL_HillClimbing_WavePeriod, W("HillClimbing_WavePeriod"), 4, ""); diff --git a/src/coreclr/vm/comthreadpool.cpp b/src/coreclr/vm/comthreadpool.cpp index 88094ff5323555..035a1f504f85bf 100644 --- a/src/coreclr/vm/comthreadpool.cpp +++ b/src/coreclr/vm/comthreadpool.cpp @@ -192,6 +192,34 @@ FCIMPL4(INT32, ThreadPoolNative::GetNextConfigUInt32Value, case 19: if (TryGetConfig(CLRConfig::INTERNAL_HillClimbing_SampleIntervalHigh, false, W("System.Threading.ThreadPool.HillClimbing.SampleIntervalHigh"))) { return 20; } FALLTHROUGH; case 20: if (TryGetConfig(CLRConfig::INTERNAL_HillClimbing_GainExponent, false, W("System.Threading.ThreadPool.HillClimbing.GainExponent"))) { return 21; } FALLTHROUGH; + case 21: + { + int threadPoolThreadTimeoutMs = g_pConfig->ThreadPoolThreadTimeoutMs(); + if (threadPoolThreadTimeoutMs >= -1) + { + *configValueRef = (UINT32)threadPoolThreadTimeoutMs; + *isBooleanRef = false; + *appContextConfigNameRef = W("System.Threading.ThreadPool.ThreadTimeoutMs"); + return 22; + } + + FALLTHROUGH; + } + + case 22: + { + int threadPoolThreadsToKeepAlive = g_pConfig->ThreadPoolThreadsToKeepAlive(); + if (threadPoolThreadsToKeepAlive >= -1) + { + *configValueRef = (UINT32)threadPoolThreadsToKeepAlive; + *isBooleanRef = false; + *appContextConfigNameRef = W("System.Threading.ThreadPool.ThreadsToKeepAlive"); + return 23; + } + + FALLTHROUGH; + } + default: *configValueRef = 0; *isBooleanRef = false; diff --git a/src/coreclr/vm/eeconfig.cpp b/src/coreclr/vm/eeconfig.cpp index 883d602b1f2309..d995220617f6ab 100644 --- a/src/coreclr/vm/eeconfig.cpp +++ b/src/coreclr/vm/eeconfig.cpp @@ -225,6 +225,9 @@ HRESULT EEConfig::Init() bDiagnosticSuspend = false; #endif + threadPoolThreadTimeoutMs = -2; // not configured + threadPoolThreadsToKeepAlive = 0; + #if defined(FEATURE_TIERED_COMPILATION) fTieredCompilation = false; fTieredCompilation_QuickJit = false; @@ -668,6 +671,19 @@ HRESULT EEConfig::sync() #endif //_DEBUG + threadPoolThreadTimeoutMs = + (int)Configuration::GetKnobDWORDValue( + W("System.Threading.ThreadPool.ThreadTimeoutMs"), + CLRConfig::EXTERNAL_ThreadPool_ThreadTimeoutMs); + threadPoolThreadsToKeepAlive = + (int)Configuration::GetKnobDWORDValue( + W("System.Threading.ThreadPool.ThreadsToKeepAlive"), + CLRConfig::EXTERNAL_ThreadPool_ThreadsToKeepAlive); + if (threadPoolThreadsToKeepAlive < -1) + { + threadPoolThreadsToKeepAlive = 0; + } + m_fInteropValidatePinnedObjects = (CLRConfig::GetConfigValue(CLRConfig::UNSUPPORTED_InteropValidatePinnedObjects) != 0); m_fInteropLogArguments = (CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_InteropLogArguments) != 0); diff --git a/src/coreclr/vm/eeconfig.h b/src/coreclr/vm/eeconfig.h index 4651cf1bc84e0f..1775180e231a32 100644 --- a/src/coreclr/vm/eeconfig.h +++ b/src/coreclr/vm/eeconfig.h @@ -469,6 +469,9 @@ class EEConfig #endif + int ThreadPoolThreadTimeoutMs() const { LIMITED_METHOD_CONTRACT; return threadPoolThreadTimeoutMs; } + int ThreadPoolThreadsToKeepAlive() const { LIMITED_METHOD_CONTRACT; return threadPoolThreadsToKeepAlive; } + private: //---------------------------------------------------------------- bool fInited; // have we synced to the registry at least once? @@ -644,6 +647,9 @@ class EEConfig DWORD testThreadAbort; #endif + int threadPoolThreadTimeoutMs; + int threadPoolThreadsToKeepAlive; + #if defined(FEATURE_TIERED_COMPILATION) bool fTieredCompilation; bool fTieredCompilation_QuickJit; diff --git a/src/coreclr/vm/win32threadpool.cpp b/src/coreclr/vm/win32threadpool.cpp index 80e5e5db5d7855..0014f2ae14602d 100644 --- a/src/coreclr/vm/win32threadpool.cpp +++ b/src/coreclr/vm/win32threadpool.cpp @@ -125,6 +125,10 @@ SPTR_IMPL(WorkRequest,ThreadpoolMgr,WorkRequestTail); // Head of work req unsigned int ThreadpoolMgr::LastCPThreadCreation=0; // last time a completion port thread was created unsigned int ThreadpoolMgr::NumberOfProcessors; // = NumberOfWorkerThreads - no. of blocked threads +DWORD ThreadpoolMgr::WorkerThreadTimeoutMs = 20 * 1000; +DWORD ThreadpoolMgr::IOCompletionThreadTimeoutMs = 15 * 1000; +int ThreadpoolMgr::NumWorkerThreadsBeingKeptAlive = 0; +int ThreadpoolMgr::NumIOCompletionThreadsBeingKeptAlive = 0; CrstStatic ThreadpoolMgr::WorkerCriticalSection; CLREvent * ThreadpoolMgr::RetiredCPWakeupEvent; // wakeup event for completion port threads @@ -344,6 +348,13 @@ BOOL ThreadpoolMgr::Initialize() NumberOfProcessors = GetCurrentProcessCpuCount(); InitPlatformVariables(); + int threadTimeoutMs = g_pConfig->ThreadPoolThreadTimeoutMs(); + if (threadTimeoutMs >= -1) + { + WorkerThreadTimeoutMs = (DWORD)threadTimeoutMs; + IOCompletionThreadTimeoutMs = (DWORD)threadTimeoutMs; + } + EX_TRY { if (!UsePortableThreadPool()) @@ -1747,6 +1758,9 @@ DWORD WINAPI ThreadpoolMgr::WorkerThreadStart(LPVOID lpArgs) ThreadCounter::Counts counts, oldCounts, newCounts; bool foundWork = true, wasNotRecalled = true; + bool isThreadKeepAliveInitialized = false; + bool keepThreadAlive = false; + counts = WorkerCounter.GetCleanCounts(); if (ETW_EVENT_ENABLED(MICROSOFT_WINDOWS_DOTNETRUNTIME_PROVIDER_DOTNET_Context, ThreadPoolWorkerThreadStart)) FireEtwThreadPoolWorkerThreadStart(counts.NumActive, counts.NumRetired, GetClrInstanceId()); @@ -1803,6 +1817,35 @@ DWORD WINAPI ThreadpoolMgr::WorkerThreadStart(LPVOID lpArgs) GCX_PREEMP_NO_DTOR(); _ASSERTE(pThread == NULL || !pThread->PreemptiveGCDisabled()); + if (!isThreadKeepAliveInitialized && fThreadInit) + { + // Determine whether to keep this thread alive. Some threads may always be kept alive based on config. + isThreadKeepAliveInitialized = true; + int threadsToKeepAlive = g_pConfig->ThreadPoolThreadsToKeepAlive(); + if (threadsToKeepAlive != 0) + { + if (threadsToKeepAlive < 0) + { + keepThreadAlive = true; + } + else + { + int count = VolatileLoadWithoutBarrier(&NumWorkerThreadsBeingKeptAlive); + while (count < threadsToKeepAlive) + { + int countBeforeUpdate = InterlockedCompareExchangeT(&NumWorkerThreadsBeingKeptAlive, count + 1, count); + if (countBeforeUpdate == count) + { + keepThreadAlive = true; + break; + } + + count = countBeforeUpdate; + } + } + } + } + // make sure there's really work. If not, go back to sleep // counts volatile read paired with CompareExchangeCounts loop set @@ -1889,7 +1932,7 @@ DWORD WINAPI ThreadpoolMgr::WorkerThreadStart(LPVOID lpArgs) while (true) { RetryRetire: - if (RetiredWorkerSemaphore->Wait(WorkerTimeout)) + if (RetiredWorkerSemaphore->Wait(keepThreadAlive ? INFINITE : WorkerThreadTimeoutMs)) { foundWork = true; @@ -1966,7 +2009,7 @@ DWORD WINAPI ThreadpoolMgr::WorkerThreadStart(LPVOID lpArgs) FireEtwThreadPoolWorkerThreadWait(counts.NumActive, counts.NumRetired, GetClrInstanceId()); RetryWaitForWork: - if (WorkerSemaphore->Wait(WorkerTimeout, WorkerThreadSpinLimit, NumberOfProcessors)) + if (WorkerSemaphore->Wait(keepThreadAlive ? INFINITE : WorkerThreadTimeoutMs, WorkerThreadSpinLimit, NumberOfProcessors)) { foundWork = true; goto Work; @@ -3080,13 +3123,13 @@ DWORD WINAPI ThreadpoolMgr::CompletionPortThreadStart(LPVOID lpArgs) PIOCompletionContext context; BOOL fIsCompletionContext; - const DWORD CP_THREAD_WAIT = 15000; /* milliseconds */ - _ASSERTE(GlobalCompletionPort != NULL); BOOL fThreadInit = FALSE; Thread *pThread = NULL; + bool isThreadKeepAliveInitialized = false; + bool keepThreadAlive = false; DWORD cpThreadWait = 0; if (g_fEEStarted) { @@ -3123,7 +3166,7 @@ DWORD WINAPI ThreadpoolMgr::CompletionPortThreadStart(LPVOID lpArgs) ThreadCounter::Counts oldCounts; ThreadCounter::Counts newCounts; - cpThreadWait = CP_THREAD_WAIT; + cpThreadWait = IOCompletionThreadTimeoutMs; for (;; ) { Top: @@ -3151,6 +3194,36 @@ DWORD WINAPI ThreadpoolMgr::CompletionPortThreadStart(LPVOID lpArgs) GCX_PREEMP_NO_DTOR(); + if (!isThreadKeepAliveInitialized && fThreadInit) + { + // Determine whether to keep this thread alive. Some threads may always be kept alive based on config. + isThreadKeepAliveInitialized = true; + int threadsToKeepAlive = g_pConfig->ThreadPoolThreadsToKeepAlive(); + if (threadsToKeepAlive != 0) + { + if (threadsToKeepAlive < 0) + { + keepThreadAlive = true; + } + else + { + int count = VolatileLoadWithoutBarrier(&NumIOCompletionThreadsBeingKeptAlive); + while (count < threadsToKeepAlive) + { + int countBeforeUpdate = + InterlockedCompareExchangeT(&NumIOCompletionThreadsBeingKeptAlive, count + 1, count); + if (countBeforeUpdate == count) + { + keepThreadAlive = true; + break; + } + + count = countBeforeUpdate; + } + } + } + } + // // We're about to wait on the IOCP; mark ourselves as no longer "working." // @@ -3165,7 +3238,7 @@ DWORD WINAPI ThreadpoolMgr::CompletionPortThreadStart(LPVOID lpArgs) // one thread listening for completions. So there's no point in having a timeout; it will // only use power unnecessarily. // - cpThreadWait = (newCounts.NumActive == 1) ? INFINITE : CP_THREAD_WAIT; + cpThreadWait = (newCounts.NumActive == 1 || keepThreadAlive) ? INFINITE : IOCompletionThreadTimeoutMs; if (oldCounts == CPThreadCounter.CompareExchangeCounts(newCounts, oldCounts)) break; diff --git a/src/coreclr/vm/win32threadpool.h b/src/coreclr/vm/win32threadpool.h index fe215efb7ced85..e7fbf1ea0037ff 100644 --- a/src/coreclr/vm/win32threadpool.h +++ b/src/coreclr/vm/win32threadpool.h @@ -971,6 +971,11 @@ class ThreadpoolMgr static unsigned int LastCPThreadCreation; // last time a completion port thread was created static unsigned int NumberOfProcessors; // = NumberOfWorkerThreads - no. of blocked threads + static DWORD WorkerThreadTimeoutMs; + static DWORD IOCompletionThreadTimeoutMs; + static int NumWorkerThreadsBeingKeptAlive; + static int NumIOCompletionThreadsBeingKeptAlive; + static BOOL IsApcPendingOnWaitThread; // Indicates if an APC is pending on the wait thread // This needs to be non-hosted, because worker threads can run prior to EE startup. @@ -980,8 +985,6 @@ class ThreadpoolMgr static CrstStatic WorkerCriticalSection; private: - static const DWORD WorkerTimeout = 20 * 1000; - DECLSPEC_ALIGN(MAX_CACHE_LINE_SIZE) SVAL_DECL(ThreadCounter,WorkerCounter); // diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.WorkerThread.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.WorkerThread.cs index 9527f84a876604..bdb3af11346699 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.WorkerThread.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.WorkerThread.cs @@ -7,6 +7,8 @@ namespace System.Threading { internal sealed partial class PortableThreadPool { + private int _numThreadsBeingKeptAlive; + /// /// The worker thread infastructure for the CLR thread pool. /// @@ -28,6 +30,22 @@ private static partial class WorkerThread // preexisting threads from running out of memory when using new stack space in low-memory situations. public const int EstimatedAdditionalStackUsagePerThreadBytes = 64 << 10; // 64 KB + private static readonly short ThreadsToKeepAlive = DetermineThreadsToKeepAlive(); + + private static short DetermineThreadsToKeepAlive() + { + const short DefaultThreadsToKeepAlive = 0; + + // The number of worker threads to keep alive after they are created. Set to -1 to keep all created worker + // threads alive. When the ThreadTimeoutMs config value is also set, for worker threads the timeout applies to + // worker threads that are in excess of the number configured for ThreadsToKeepAlive. + short threadsToKeepAlive = + AppContextConfigHelper.GetInt16Config( + "System.Threading.ThreadPool.ThreadsToKeepAlive", + DefaultThreadsToKeepAlive); + return threadsToKeepAlive >= -1 ? threadsToKeepAlive : DefaultThreadsToKeepAlive; + } + /// /// Semaphore for controlling how many threads are currently working. /// @@ -65,10 +83,36 @@ private static void WorkerThreadStart() LowLevelLock threadAdjustmentLock = threadPoolInstance._threadAdjustmentLock; LowLevelLifoSemaphore semaphore = s_semaphore; + // Determine the idle timeout to use for this thread. Some threads may always be kept alive based on config. + int timeoutMs = ThreadPoolThreadTimeoutMs; + if (ThreadsToKeepAlive != 0) + { + if (ThreadsToKeepAlive < 0) + { + timeoutMs = Timeout.Infinite; + } + else + { + int count = threadPoolInstance._numThreadsBeingKeptAlive; + while (count < ThreadsToKeepAlive) + { + int countBeforeUpdate = + Interlocked.CompareExchange(ref threadPoolInstance._numThreadsBeingKeptAlive, count + 1, count); + if (countBeforeUpdate == count) + { + timeoutMs = Timeout.Infinite; + break; + } + + count = countBeforeUpdate; + } + } + } + while (true) { bool spinWait = true; - while (semaphore.Wait(ThreadPoolThreadTimeoutMs, spinWait)) + while (semaphore.Wait(timeoutMs, spinWait)) { bool alreadyRemovedWorkingWorker = false; while (TakeActiveRequest(threadPoolInstance)) diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.cs index 57f25de034c249..177acd02a219ad 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.cs @@ -13,7 +13,6 @@ namespace System.Threading /// internal sealed partial class PortableThreadPool { - private const int ThreadPoolThreadTimeoutMs = 20 * 1000; // If you change this make sure to change the timeout times in the tests. private const int SmallStackSizeBytes = 256 * 1024; private const short MaxPossibleThreadCount = short.MaxValue; @@ -34,6 +33,22 @@ internal sealed partial class PortableThreadPool private static readonly short ForcedMaxWorkerThreads = AppContextConfigHelper.GetInt16Config("System.Threading.ThreadPool.MaxThreads", 0, false); + private static readonly int ThreadPoolThreadTimeoutMs = DetermineThreadPoolThreadTimeoutMs(); + + private static int DetermineThreadPoolThreadTimeoutMs() + { + const int DefaultThreadPoolThreadTimeoutMs = 20 * 1000; // If you change this make sure to change the timeout times in the tests. + + // The amount of time in milliseconds a thread pool thread waits without having done any work before timing out and + // exiting. Set to -1 to disable the timeout. Applies to worker threads and wait threads. Also see the + // ThreadsToKeepAlive config value for relevant information. + int timeoutMs = + AppContextConfigHelper.GetInt32Config( + "System.Threading.ThreadPool.ThreadTimeoutMs", + DefaultThreadPoolThreadTimeoutMs); + return timeoutMs >= -1 ? timeoutMs : DefaultThreadPoolThreadTimeoutMs; + } + [ThreadStatic] private static object? t_completionCountObject; From 5d8b3d62c9ff22921d7327fe4366a4b1ea890c8f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Oct 2023 10:01:26 -0700 Subject: [PATCH 09/17] Fix max chunk size limiting (#81607) When the new stub heaps were implemented, the chunk size needed to be limited so that all precodes for a chunk fit into a single memory page. That was done in `MethodDescChunk::CreateChunk`. But it was discovered now that there is another place where it needs to be limited, the `MethodTableBuilder::AllocAndInitMethodDescChunk`. The JIT\opt\ObjectStackAllocation\ObjectStackAllocationTests started to fail in the outerloop due to too long chunk. The failure happens in crossgen2 as it is using a separate build of runtime in release and only that uncovers the problem. And only when DOTNET_TieredCompilation=0 and DOTNET_ProfApi_RejitOnAttach=0. This change fixes the problem. With this change applied to the dotnet runtime version used by the crossgen2 and patching it in place in the .dotnet/shared/.... directory, the issue doesn't occur. Close #81103 Co-authored-by: Jan Vorlicek --- src/coreclr/vm/methodtablebuilder.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/coreclr/vm/methodtablebuilder.cpp b/src/coreclr/vm/methodtablebuilder.cpp index 245fcaf55bf2bb..3a941210b7a209 100644 --- a/src/coreclr/vm/methodtablebuilder.cpp +++ b/src/coreclr/vm/methodtablebuilder.cpp @@ -6871,9 +6871,15 @@ VOID MethodTableBuilder::AllocAndInitMethodDescs() SIZE_T sizeOfMethodDescs = 0; // current running size of methodDesc chunk int startIndex = 0; // start of the current chunk (index into bmtMethod array) + // Limit the maximum MethodDescs per chunk by the number of precodes that can fit to a single memory page, + // since we allocate consecutive temporary entry points for all MethodDescs in the whole chunk. + DWORD maxPrecodesPerPage = Precode::GetMaxTemporaryEntryPointsCount(); + DWORD methodDescCount = 0; + DeclaredMethodIterator it(*this); while (it.Next()) { + DWORD currentSlotMethodDescCount = 1; int tokenRange = GetTokenRange(it.Token()); // This code assumes that iterator returns tokens in ascending order. If this assumption does not hold, @@ -6896,6 +6902,7 @@ VOID MethodTableBuilder::AllocAndInitMethodDescs() // See comment in AllocAndInitMethodDescChunk if (NeedsTightlyBoundUnboxingStub(*it)) { + currentSlotMethodDescCount = 2; size *= 2; if (bmtGenerics->GetNumGenericArgs() == 0) { @@ -6907,7 +6914,8 @@ VOID MethodTableBuilder::AllocAndInitMethodDescs() } if (tokenRange != currentTokenRange || - sizeOfMethodDescs + size > MethodDescChunk::MaxSizeOfMethodDescs) + sizeOfMethodDescs + size > MethodDescChunk::MaxSizeOfMethodDescs || + methodDescCount + currentSlotMethodDescCount > maxPrecodesPerPage) { if (sizeOfMethodDescs != 0) { @@ -6917,9 +6925,11 @@ VOID MethodTableBuilder::AllocAndInitMethodDescs() currentTokenRange = tokenRange; sizeOfMethodDescs = 0; + methodDescCount = 0; } sizeOfMethodDescs += size; + methodDescCount += currentSlotMethodDescCount; } if (sizeOfMethodDescs != 0) From e5c41ae3af010f3ec71f6d37f487a5d621cb056c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20S=C3=A1nchez=20L=C3=B3pez?= <1175054+carlossanlop@users.noreply.github.com> Date: Thu, 12 Oct 2023 17:50:55 -0600 Subject: [PATCH 10/17] Re-add removed feed --- NuGet.config | 1 + 1 file changed, 1 insertion(+) diff --git a/NuGet.config b/NuGet.config index f9a9c97b16b8b6..ecdeb175256d71 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,6 +9,7 @@ + From cbe1474bda232896d6ab5d9ffe4cff6c50ea140a Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 13 Oct 2023 16:34:09 -0600 Subject: [PATCH 11/17] Update dependencies from https://github.com/dotnet/xharness build 20230905.3 (#93392) Microsoft.DotNet.XHarness.CLI , Microsoft.DotNet.XHarness.TestRunners.Common , Microsoft.DotNet.XHarness.TestRunners.Xunit From Version 7.0.0-prerelease.23407.3 -> To Version 7.0.0-prerelease.23455.3 Co-authored-by: dotnet-maestro[bot] From 69483a0994c0fa42a48a01bca3f7af2f45e08d86 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 16 Oct 2023 11:27:19 -0700 Subject: [PATCH 12/17] Workaround for https://github.com/dotnet/runtime/issues/93442 (#93529) Co-authored-by: Jan Kotas --- src/coreclr/inc/safemath.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/coreclr/inc/safemath.h b/src/coreclr/inc/safemath.h index 3f6d5c5716bdb4..fcd51af3de8cb0 100644 --- a/src/coreclr/inc/safemath.h +++ b/src/coreclr/inc/safemath.h @@ -688,6 +688,10 @@ template class ClrSafeInt INDEBUG( mutable bool m_checkedOverflow; ) }; +#if defined(_MSC_VER) && defined(HOST_ARM64) // Workaround for https://github.com/dotnet/runtime/issues/93442 +#pragma optimize("", off) +#endif + template <> inline bool ClrSafeInt::multiply(int64_t lhs, int64_t rhs, int64_t &result) { @@ -874,6 +878,10 @@ inline bool ClrSafeInt::multiply(uint8_t lhs, uint8_t rhs, uint8_t &res return true; } +#if defined(_MSC_VER) && defined(HOST_ARM64) // Workaround for https://github.com/dotnet/runtime/issues/93442 +#pragma optimize("", on) +#endif + // Allows creation of a ClrSafeInt corresponding to the type of the argument. template ClrSafeInt AsClrSafeInt(T t) From d1ca532a244d6784db2d3d77dc3b4a3acfac687c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 16 Oct 2023 13:08:55 -0700 Subject: [PATCH 13/17] TcpReceiveSendGetsCanceledByDispose: update test for change in Linux kernel. (#93505) Co-authored-by: Tom Deseyn --- .../tests/FunctionalTests/SendReceive/SendReceive.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendReceive/SendReceive.cs b/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendReceive/SendReceive.cs index a71f4d5c35627a..94deadf22badf0 100644 --- a/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendReceive/SendReceive.cs +++ b/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendReceive/SendReceive.cs @@ -1013,11 +1013,12 @@ public async Task TcpReceiveSendGetsCanceledByDispose(bool receiveOrSend, bool i return; } - // RHEL7 kernel has a bug preventing close(AF_UNKNOWN) to succeed with IPv6 sockets. - // In this case Dispose will trigger a graceful shutdown, which means that receive will succeed on socket2. - // This bug is fixed in kernel 3.10.0-1160.25+. - // TODO: Remove this, once CI machines are updated to a newer kernel. - bool mayShutdownGraceful = UsesSync && PlatformDetection.IsRedHatFamily7 && receiveOrSend && (ipv6Server || dualModeClient); + // .NET uses connect(AF_UNSPEC) to abort on-going operations on Linux. + // Linux 6.4+ introduced a change (4faeee0cf8a5d88d63cdbc3bab124fb0e6aed08c) which disallows + // this operation while operations are on-going. + // When the connect fails, .NET falls back to use shutdown(SHUT_RDWR). + // This causes the receive on socket2 to succeed instead of failing with ConnectionReset. + bool mayShutdownGraceful = UsesSync && PlatformDetection.IsLinux && receiveOrSend; // We try this a couple of times to deal with a timing race: if the Dispose happens // before the operation is started, the peer won't see a ConnectionReset SocketException and we won't From 4b2416f3345b6ffe9e2534303c9d494fb493ef78 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 16 Oct 2023 13:10:11 -0700 Subject: [PATCH 14/17] [release/7.0-staging] Update dependencies from dotnet/arcade dotnet/icu dotnet/emsdk (#93390) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update dependencies from https://github.com/dotnet/arcade build 20231011.9 Microsoft.DotNet.ApiCompat , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Build.Tasks.Archives , Microsoft.DotNet.Build.Tasks.Feed , Microsoft.DotNet.Build.Tasks.Installers , Microsoft.DotNet.Build.Tasks.Packaging , Microsoft.DotNet.Build.Tasks.TargetFramework , Microsoft.DotNet.Build.Tasks.Templating , Microsoft.DotNet.Build.Tasks.Workloads , Microsoft.DotNet.CodeAnalysis , Microsoft.DotNet.GenAPI , Microsoft.DotNet.GenFacades , Microsoft.DotNet.Helix.Sdk , Microsoft.DotNet.PackageTesting , Microsoft.DotNet.RemoteExecutor , Microsoft.DotNet.SharedFramework.Sdk , Microsoft.DotNet.VersionTools.Tasks , Microsoft.DotNet.XUnitConsoleRunner , Microsoft.DotNet.XUnitExtensions From Version 7.0.0-beta.23408.3 -> To Version 7.0.0-beta.23511.9 * Update dependencies from https://github.com/dotnet/icu build 20231012.1 Microsoft.NETCore.Runtime.ICU.Transport From Version 7.0.0-rtm.23409.2 -> To Version 7.0.0-rtm.23512.1 * Update dependencies from https://github.com/dotnet/emsdk build 20231012.1 Microsoft.NET.Workload.Emscripten.net6.Manifest-7.0.100 , Microsoft.NET.Workload.Emscripten.net7.Manifest-7.0.100 From Version 7.0.13 -> To Version 7.0.13 --------- Co-authored-by: dotnet-maestro[bot] Co-authored-by: Carlos Sánchez López <1175054+carlossanlop@users.noreply.github.com> --- eng/Version.Details.xml | 4 ++-- eng/Versions.props | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 2bda793c257990..090b17a0e02f1e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,8 +1,8 @@ - + https://github.com/dotnet/icu - d7b56b550d4e8319a6851ad2f2544450d521e0ed + 17f1c7cf5247e5ff4d344591ea966334a4f99ff5 https://github.com/dotnet/msquic diff --git a/eng/Versions.props b/eng/Versions.props index 537d2fbac11fb2..20bdebb1e342c4 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -177,7 +177,7 @@ 7.0.100-1.23401.1 $(MicrosoftNETILLinkTasksVersion) - 7.0.0-rtm.23409.2 + 7.0.0-rtm.23512.1 2.2.3 7.0.0-alpha.1.22459.1 From 317a01fd990780e7c9ff640a0b456dee5459fea6 Mon Sep 17 00:00:00 2001 From: Thays Grazia Date: Mon, 16 Oct 2023 17:14:34 -0300 Subject: [PATCH 15/17] [release/7.0] Fixes not find assembly and pdb if it's not using the default path (#92955) * Fixing https://github.com/dotnet/aspnetcore/issues/51093 * Addressing comments * Addressing comments * Addressing comments --- .../debugger/BrowserDebugProxy/DebugStore.cs | 62 ++++++++----------- 1 file changed, 26 insertions(+), 36 deletions(-) diff --git a/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs b/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs index 5a2b3d2577f469..cdd839648b379a 100644 --- a/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs +++ b/src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs @@ -1491,50 +1491,24 @@ public async IAsyncEnumerable Load(SessionId id, string[] loaded_fil foreach (string url in asm_files) { - try - { - string candidate_pdb = Path.ChangeExtension(url, "pdb"); - string pdb = pdb_files.FirstOrDefault(n => n == candidate_pdb); + string candidate_pdb = Path.ChangeExtension(url, "pdb"); + string pdb = pdb_files.FirstOrDefault(n => n == candidate_pdb); - steps.Add( - new DebugItem - { - Url = url, - Data = Task.WhenAll(MonoProxy.HttpClient.GetByteArrayAsync(url, token), pdb != null ? MonoProxy.HttpClient.GetByteArrayAsync(pdb, token) : Task.FromResult(null)) - }); - } - catch (Exception e) - { - if (tryUseDebuggerProtocol) - { - try - { - string unescapedFileName = Uri.UnescapeDataString(url); - steps.Add( - new DebugItem - { - Url = url, - Data = context.SdbAgent.GetBytesFromAssemblyAndPdb(Path.GetFileName(unescapedFileName), token) - }); - } - catch (Exception ex) - { - logger.LogDebug($"Failed to get bytes using debugger protocol {url} ({ex.Message})"); - } - } - else + steps.Add( + new DebugItem { - logger.LogDebug($"Failed to read {url} ({e.Message})"); - } - } + Url = url, + Data = Task.WhenAll(MonoProxy.HttpClient.GetByteArrayAsync(url, token), pdb != null ? MonoProxy.HttpClient.GetByteArrayAsync(pdb, token) : Task.FromResult(null)) + }); } foreach (DebugItem step in steps) { AssemblyInfo assembly = null; + byte[][] bytes; try { - byte[][] bytes = await step.Data.ConfigureAwait(false); + bytes = await step.Data.ConfigureAwait(false); if (bytes[0] == null) { logger.LogDebug($"Bytes from assembly {step.Url} is NULL"); @@ -1544,7 +1518,23 @@ public async IAsyncEnumerable Load(SessionId id, string[] loaded_fil } catch (Exception e) { - logger.LogError($"Failed to load {step.Url} ({e.Message})"); + try + { + if (tryUseDebuggerProtocol) + { + string unescapedFileName = Uri.UnescapeDataString(step.Url); + bytes = await context.SdbAgent.GetBytesFromAssemblyAndPdb(Path.GetFileName(unescapedFileName), token).ConfigureAwait(false); + assembly = new AssemblyInfo(monoProxy, id, step.Url, bytes[0], bytes[1], logger, token); + } + else + { + logger.LogDebug($"Failed to read {step.Url} ({e})"); + } + } + catch (Exception ex) + { + logger.LogError($"Failed to load {step.Url} ({ex})"); + } } if (assembly == null) continue; From 19045fadf2662005ac89a664dc2539aa0ee3d7ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20K=C3=B6plinger?= Date: Mon, 16 Oct 2023 22:17:08 +0200 Subject: [PATCH 16/17] [release/7.0][mono] Backport offsets-tool fixes for Xcode 15 compatibility (#93543) Fixes https://github.com/dotnet/runtime/issues/93009 --- src/mono/mono/tools/offsets-tool/clang/cindex.py | 11 ++++++++++- src/mono/mono/tools/offsets-tool/offsets-tool.py | 10 ++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/mono/mono/tools/offsets-tool/clang/cindex.py b/src/mono/mono/tools/offsets-tool/clang/cindex.py index 44c6f49096f495..6b9da164596609 100644 --- a/src/mono/mono/tools/offsets-tool/clang/cindex.py +++ b/src/mono/mono/tools/offsets-tool/clang/cindex.py @@ -646,6 +646,11 @@ def name(self): @classmethod def from_id(cls, id): + if cls == CursorKind and id == 300: + # --- DOTNET change --- + # The id of CursorKind.TRANSLATION_UNIT changed in https://github.com/llvm/llvm-project/commit/bb83f8e70bd1d56152f02307adacd718cd67e312, + # add mapping from the old to the new value so using the binding with an older clang still works. + return cls._kinds[350] if id >= len(cls._kinds) or cls._kinds[id] is None: raise ValueError('Unknown template argument kind %d' % id) return cls._kinds[id] @@ -1312,7 +1317,7 @@ def __repr__(self): # # The translation unit cursor exists primarily to act as the root cursor for # traversing the contents of a translation unit. -CursorKind.TRANSLATION_UNIT = CursorKind(300) +CursorKind.TRANSLATION_UNIT = CursorKind(350) ### # Attributes @@ -1342,6 +1347,8 @@ def __repr__(self): CursorKind.DLLEXPORT_ATTR = CursorKind(418) CursorKind.DLLIMPORT_ATTR = CursorKind(419) +# Temporary fake value to work around xcode 15 beta / clang 15 +CursorKind.XCODEBETA_ATTR = CursorKind(437) CursorKind.CONVERGENT_ATTR = CursorKind(438) CursorKind.WARN_UNUSED_ATTR = CursorKind(439) CursorKind.WARN_UNUSED_RESULT_ATTR = CursorKind(440) @@ -2059,6 +2066,7 @@ def __repr__(self): TypeKind.OBJCSEL = TypeKind(29) TypeKind.FLOAT128 = TypeKind(30) TypeKind.HALF = TypeKind(31) +TypeKind.IBM128 = TypeKind(40) TypeKind.COMPLEX = TypeKind(100) TypeKind.POINTER = TypeKind(101) TypeKind.BLOCKPOINTER = TypeKind(102) @@ -2122,6 +2130,7 @@ def __repr__(self): TypeKind.OCLRESERVEID = TypeKind(160) TypeKind.EXTVECTOR = TypeKind(176) +TypeKind.ATOMIC = TypeKind(177) class RefQualifierKind(BaseEnumeration): """Describes a specific ref-qualifier of a type.""" diff --git a/src/mono/mono/tools/offsets-tool/offsets-tool.py b/src/mono/mono/tools/offsets-tool/offsets-tool.py index 4d3a6ccef91da9..b82f205d562021 100644 --- a/src/mono/mono/tools/offsets-tool/offsets-tool.py +++ b/src/mono/mono/tools/offsets-tool/offsets-tool.py @@ -365,8 +365,11 @@ def gen (self): if type.size == -1: continue f.write ("DECL_SIZE2(%s,%s)\n" % (type.name, type.size)) + done_fields = {} for field in type.fields: - f.write ("DECL_OFFSET2(%s,%s,%s)\n" % (type.name, field.name, field.offset)) + if field.name not in done_fields: + f.write ("DECL_OFFSET2(%s,%s,%s)\n" % (type.name, field.name, field.offset)) + done_fields [field.name] = field.name f.write ("#endif //disable metadata check\n") f.write ("#ifndef DISABLE_JIT_OFFSETS\n") @@ -376,8 +379,11 @@ def gen (self): if type.size == -1: continue f.write ("DECL_SIZE2(%s,%s)\n" % (type.name, type.size)) + done_fields = {} for field in type.fields: - f.write ("DECL_OFFSET2(%s,%s,%s)\n" % (type.name, field.name, field.offset)) + if field.name not in done_fields: + f.write ("DECL_OFFSET2(%s,%s,%s)\n" % (type.name, field.name, field.offset)) + done_fields [field.name] = field.name f.write ("#endif //disable jit check\n") f.write ("#endif //cross compiler checks\n") From 72a4b2667cb3f2cbf30dadcf436cb90e7958dc99 Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Mon, 16 Oct 2023 17:46:15 -0500 Subject: [PATCH 17/17] ProcessTests: allow WorkingSet to be zero just after launching the process. (#85649) (#93572) * ProcessTests: allow WorkingSet to be zero just after launching the process. The WorkingSet tests fail on Fedora 38+ because a zero working set value is observed just after the process start. Co-authored-by: Tom Deseyn --- .../tests/ProcessTests.cs | 82 ++++++++++++++++++- 1 file changed, 78 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessTests.cs index 80fa5b8bb75b45..881a72f78fe4a6 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessTests.cs @@ -766,7 +766,25 @@ public void TestPeakWorkingSet64() { CreateDefaultProcess(); - AssertNonZeroAllZeroDarwin(_process.PeakWorkingSet64); + if (OperatingSystem.IsMacOS()) + { + Assert.Equal(0, _process.PeakWorkingSet64); + return; + } + + // On recent Linux kernels (6.2+) working set can be zero just after the process started. + ExecuteWithRetryOnLinux(() => + { + try + { + Assert.NotEqual(0, _process.PeakWorkingSet64); + } + catch + { + _process.Refresh(); + throw; + } + }); } [Fact] @@ -819,7 +837,19 @@ public void TestWorkingSet64() return; } - Assert.InRange(_process.WorkingSet64, 1, long.MaxValue); + // On recent Linux kernels (6.2+) working set can be zero just after the process started. + ExecuteWithRetryOnLinux(() => + { + try + { + Assert.InRange(_process.WorkingSet64, 1, long.MaxValue); + } + catch + { + _process.Refresh(); + throw; + } + }); } [Fact] @@ -2011,9 +2041,29 @@ public void TestPeakWorkingSet() { CreateDefaultProcess(); + if (OperatingSystem.IsMacOS()) + { #pragma warning disable 0618 - AssertNonZeroAllZeroDarwin(_process.PeakWorkingSet); + Assert.Equal(0, _process.PeakWorkingSet); #pragma warning restore 0618 + return; + } + + // On recent Linux kernels (6.2+) working set can be zero just after the process started. + ExecuteWithRetryOnLinux(() => + { + try + { +#pragma warning disable 0618 + Assert.NotEqual(0, _process.PeakWorkingSet); +#pragma warning restore 0618 + } + catch + { + _process.Refresh(); + throw; + } + }); } [Fact] @@ -2077,9 +2127,21 @@ public void TestWorkingSet() return; } + // On recent Linux kernels (6.2+) working set can be zero just after the process started. + ExecuteWithRetryOnLinux(() => + { + try + { #pragma warning disable 0618 - Assert.InRange(_process.WorkingSet, 1, int.MaxValue); + Assert.InRange(_process.WorkingSet, 1, int.MaxValue); #pragma warning restore 0618 + } + catch + { + _process.Refresh(); + throw; + } + }); } [Fact] @@ -2673,5 +2735,17 @@ private SecureString AsSecureString(string str) return secureString; } + + private static void ExecuteWithRetryOnLinux(Action test) + { + if (OperatingSystem.IsLinux()) + { + RetryHelper.Execute(test, retryWhen: ex => ex is XunitException); + } + else + { + test(); + } + } } }