From 67b110e8b64a101c81f6f8b9746b0648bceb2cf4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 23 Dec 2021 15:42:38 -0700 Subject: [PATCH 01/39] [release/6.0] Fixing a possible null reference error in WebSocket deflate. (#62716) * Fixing a possible null reference error in websocket deflate functionality. * Style feedback. Co-authored-by: Ivan Zlatanov --- .../Net/WebSockets/Compression/WebSocketDeflater.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Net.WebSockets/src/System/Net/WebSockets/Compression/WebSocketDeflater.cs b/src/libraries/System.Net.WebSockets/src/System/Net/WebSockets/Compression/WebSocketDeflater.cs index 7d8eb832c2e2ad..7c8db8eedcc871 100644 --- a/src/libraries/System.Net.WebSockets/src/System/Net/WebSockets/Compression/WebSocketDeflater.cs +++ b/src/libraries/System.Net.WebSockets/src/System/Net/WebSockets/Compression/WebSocketDeflater.cs @@ -144,7 +144,13 @@ private unsafe void UnsafeDeflate(ReadOnlySpan input, Span output, o consumed = input.Length - (int)_stream.AvailIn; written = output.Length - (int)_stream.AvailOut; - needsMoreBuffer = errorCode == ErrorCode.BufError || _stream.AvailIn > 0; + // It is important here to also check that we haven't + // exhausted the output buffer because after deflating we're + // always going to issue a flush and a flush with empty output + // is going to throw. + needsMoreBuffer = errorCode == ErrorCode.BufError + || _stream.AvailIn > 0 + || written == output.Length; } } @@ -152,6 +158,7 @@ private unsafe int UnsafeFlush(Span output, out bool needsMoreBuffer) { Debug.Assert(_stream is not null); Debug.Assert(_stream.AvailIn == 0); + Debug.Assert(output.Length > 0); fixed (byte* fixedOutput = output) { From 3b71fd8b7b5b8ddf7aeece6a7296f408b7e1fd1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Cant=C3=BA?= Date: Thu, 30 Dec 2021 20:10:51 -0800 Subject: [PATCH 02/39] [release/6.0] Don't create multiple large files at the same time (#63032) * Move NoInt32OverflowInTheBufferingLogic to OuterLoop and address pending feedback (#60606) * Don't create multiple large files at the same time (#62519) * move existing large file tests into a separate type (no code changes) * don't run the large file tests in parallel * use FileOptions.DeleteOnClose to ensure that each test removes it's own file use single large file to test File.ReadAllBytes and ile.ReadAllBytesAsync for both limits * Fix DisableParallelization build issue * Apply suggestions from code review Co-authored-by: Adam Sitnik * Update src/libraries/System.IO.FileSystem/tests/LargeFileTests.cs Co-authored-by: Adam Sitnik * Notepad with uppercase (#62487) Co-authored-by: Adam Sitnik Co-authored-by: Dan Moseley --- .../tests/ProcessStartInfoTests.cs | 2 +- .../tests/ProcessTests.cs | 15 +++-- .../tests/File/ReadWriteAllBytes.cs | 15 ----- .../tests/File/ReadWriteAllBytesAsync.cs | 16 ----- .../tests/FileStream/Read.cs | 44 ------------- .../tests/LargeFileTests.cs | 64 +++++++++++++++++++ .../tests/System.IO.FileSystem.Tests.csproj | 1 + .../BufferedStream/BufferedStreamTests.cs | 34 ++++++++++ 8 files changed, 110 insertions(+), 81 deletions(-) create mode 100644 src/libraries/System.IO.FileSystem/tests/LargeFileTests.cs diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs index 38062773cb3671..6f304b08989f55 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessStartInfoTests.cs @@ -1318,7 +1318,7 @@ private void VerifyNotepadMainWindowTitle(Process process, string filename) string expected = Path.GetFileNameWithoutExtension(filename); process.WaitForInputIdle(); // Give the file a chance to load - Assert.Equal("notepad", process.ProcessName); + Assert.Equal("notepad", process.ProcessName.ToLower()); // Notepad calls CreateWindowEx with pWindowName of empty string, then calls SetWindowTextW // with "Untitled - Notepad" then finally if you're opening a file, calls SetWindowTextW diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessTests.cs index 880da43a646539..71e27ea0d9cf9e 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessTests.cs @@ -267,11 +267,16 @@ public void ProcessStart_UseShellExecute_OnWindows_DoesNotThrow(bool isFolder) { if (px != null) // sometimes process is null { - Assert.Equal("notepad", px.ProcessName); - - px.Kill(); - Assert.True(px.WaitForExit(WaitInMS)); - px.WaitForExit(); // wait for event handlers to complete + try + { + Assert.Equal("notepad", px.ProcessName.ToLower()); + } + finally + { + px.Kill(); + Assert.True(px.WaitForExit(WaitInMS)); + px.WaitForExit(); // wait for event handlers to complete + } } } } diff --git a/src/libraries/System.IO.FileSystem/tests/File/ReadWriteAllBytes.cs b/src/libraries/System.IO.FileSystem/tests/File/ReadWriteAllBytes.cs index 5dd2e3f1bda6d7..a7815dd40eb126 100644 --- a/src/libraries/System.IO.FileSystem/tests/File/ReadWriteAllBytes.cs +++ b/src/libraries/System.IO.FileSystem/tests/File/ReadWriteAllBytes.cs @@ -57,21 +57,6 @@ public void ValidWrite(int size) File.Delete(path); } - [Fact] - [OuterLoop] - [ActiveIssue("https://github.com/dotnet/runtime/issues/45954", TestPlatforms.Browser)] - public void ReadFileOver2GB() - { - string path = GetTestFilePath(); - using (FileStream fs = File.Create(path)) - { - fs.SetLength(int.MaxValue + 1L); - } - - // File is too large for ReadAllBytes at once - Assert.Throws(() => File.ReadAllBytes(path)); - } - [Fact] public void Overwrite() { diff --git a/src/libraries/System.IO.FileSystem/tests/File/ReadWriteAllBytesAsync.cs b/src/libraries/System.IO.FileSystem/tests/File/ReadWriteAllBytesAsync.cs index be562f15ca953a..c4f147cbb80092 100644 --- a/src/libraries/System.IO.FileSystem/tests/File/ReadWriteAllBytesAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/File/ReadWriteAllBytesAsync.cs @@ -6,7 +6,6 @@ using System.Threading.Tasks; using Xunit; using System.IO.Pipes; -using Microsoft.DotNet.XUnitExtensions; namespace System.IO.Tests { @@ -70,21 +69,6 @@ public Task AlreadyCanceledAsync() async () => await File.WriteAllBytesAsync(path, new byte[0], token)); } - [Fact] - [OuterLoop] - [ActiveIssue("https://github.com/dotnet/runtime/issues/45954", TestPlatforms.Browser)] - public Task ReadFileOver2GBAsync() - { - string path = GetTestFilePath(); - using (FileStream fs = File.Create(path)) - { - fs.SetLength(int.MaxValue + 1L); - } - - // File is too large for ReadAllBytes at once - return Assert.ThrowsAsync(async () => await File.ReadAllBytesAsync(path)); - } - [Fact] public async Task OverwriteAsync() { diff --git a/src/libraries/System.IO.FileSystem/tests/FileStream/Read.cs b/src/libraries/System.IO.FileSystem/tests/FileStream/Read.cs index fec3a7a84e2f14..b6c555920bca3b 100644 --- a/src/libraries/System.IO.FileSystem/tests/FileStream/Read.cs +++ b/src/libraries/System.IO.FileSystem/tests/FileStream/Read.cs @@ -14,49 +14,5 @@ public void NegativeReadRootThrows() Assert.Throws(() => new FileStream(Path.GetPathRoot(Directory.GetCurrentDirectory()), FileMode.Open, FileAccess.Read)); } - - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.Is64BitProcess))] - [ActiveIssue("https://github.com/dotnet/runtime/issues/45954", TestPlatforms.Browser)] - public void NoInt32OverflowInTheBufferingLogic() - { - const long position1 = 10; - const long position2 = (1L << 32) + position1; - - string filePath = GetTestFilePath(); - byte[] data1 = new byte[] { 1, 2, 3, 4, 5 }; - byte[] data2 = new byte[] { 6, 7, 8, 9, 10 }; - byte[] buffer = new byte[5]; - - using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write)) - { - stream.Seek(position1, SeekOrigin.Begin); - stream.Write(data1, 0, data1.Length); - - stream.Seek(position2, SeekOrigin.Begin); - stream.Write(data2, 0, data2.Length); - } - - using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read)) - { - stream.Seek(position1, SeekOrigin.Begin); - Assert.Equal(buffer.Length, stream.Read(buffer)); - Assert.Equal(data1, buffer); - - stream.Seek(position2, SeekOrigin.Begin); - Assert.Equal(buffer.Length, stream.Read(buffer)); - Assert.Equal(data2, buffer); - } - - using (var stream = new BufferedStream(new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.None, bufferSize: 0))) - { - stream.Seek(position1, SeekOrigin.Begin); - Assert.Equal(buffer.Length, stream.Read(buffer)); - Assert.Equal(data1, buffer); - - stream.Seek(position2, SeekOrigin.Begin); - Assert.Equal(buffer.Length, stream.Read(buffer)); - Assert.Equal(data2, buffer); - } - } } } diff --git a/src/libraries/System.IO.FileSystem/tests/LargeFileTests.cs b/src/libraries/System.IO.FileSystem/tests/LargeFileTests.cs new file mode 100644 index 00000000000000..c8158f8b97784a --- /dev/null +++ b/src/libraries/System.IO.FileSystem/tests/LargeFileTests.cs @@ -0,0 +1,64 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.IO.Tests; +using System.Threading.Tasks; +using Xunit; + +namespace System.IO.FileSystem.Tests +{ + [OuterLoop] + [ActiveIssue("https://github.com/dotnet/runtime/issues/45954", TestPlatforms.Browser)] + [Collection(nameof(NoParallelTests))] // don't create multiple large files at the same time + public class LargeFileTests : FileSystemTest + { + [Fact] + public async Task ReadAllBytesOverLimit() + { + using FileStream fs = new (GetTestFilePath(), FileMode.Create, FileAccess.Write, FileShare.Read, 4096, FileOptions.DeleteOnClose); + + foreach (long lengthOverLimit in new long[] { int.MaxValue + 1L }) + { + fs.SetLength(lengthOverLimit); + + Assert.Throws(() => File.ReadAllBytes(fs.Name)); + await Assert.ThrowsAsync(async () => await File.ReadAllBytesAsync(fs.Name)); + } + } + + [Fact] + public void NoInt32OverflowInTheBufferingLogic() + { + const long position1 = 10; + const long position2 = (1L << 32) + position1; + + string filePath = GetTestFilePath(); + byte[] data1 = new byte[] { 1, 2, 3, 4, 5 }; + byte[] data2 = new byte[] { 6, 7, 8, 9, 10 }; + byte[] buffer = new byte[5]; + + using (FileStream stream = File.Create(filePath)) + { + stream.Seek(position1, SeekOrigin.Begin); + stream.Write(data1); + + stream.Seek(position2, SeekOrigin.Begin); + stream.Write(data2); + } + + using (FileStream stream = new (filePath, FileMode.Open, FileAccess.Read, FileShare.None, 4096, FileOptions.DeleteOnClose)) + { + stream.Seek(position1, SeekOrigin.Begin); + Assert.Equal(buffer.Length, stream.Read(buffer)); + Assert.Equal(data1, buffer); + + stream.Seek(position2, SeekOrigin.Begin); + Assert.Equal(buffer.Length, stream.Read(buffer)); + Assert.Equal(data2, buffer); + } + } + } + + [CollectionDefinition(nameof(NoParallelTests), DisableParallelization = true)] + public partial class NoParallelTests { } +} diff --git a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj index ea17b396daa305..1ffb30f887c246 100644 --- a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj +++ b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj @@ -55,6 +55,7 @@ + diff --git a/src/libraries/System.IO/tests/BufferedStream/BufferedStreamTests.cs b/src/libraries/System.IO/tests/BufferedStream/BufferedStreamTests.cs index 08268755202054..8d11c80336711a 100644 --- a/src/libraries/System.IO/tests/BufferedStream/BufferedStreamTests.cs +++ b/src/libraries/System.IO/tests/BufferedStream/BufferedStreamTests.cs @@ -323,6 +323,40 @@ public async Task CopyToTest_ReadBeforeCopy_CopiesAllData(bool copyAsynchronousl Array.Copy(data, 1, expected, 0, expected.Length); Assert.Equal(expected, dst.ToArray()); } + + [Fact] + [OuterLoop] + [ActiveIssue("https://github.com/dotnet/runtime/issues/45954", TestPlatforms.Browser)] + public void NoInt32OverflowInTheBufferingLogic() + { + const long position1 = 10; + const long position2 = (1L << 32) + position1; + + string filePath = Path.GetTempFileName(); + byte[] data1 = new byte[] { 1, 2, 3, 4, 5 }; + byte[] data2 = new byte[] { 6, 7, 8, 9, 10 }; + byte[] buffer = new byte[5]; + + using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write)) + { + stream.Seek(position1, SeekOrigin.Begin); + stream.Write(data1); + + stream.Seek(position2, SeekOrigin.Begin); + stream.Write(data2); + } + + using (var stream = new BufferedStream(new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.None, bufferSize: 0, FileOptions.DeleteOnClose))) + { + stream.Seek(position1, SeekOrigin.Begin); + Assert.Equal(buffer.Length, stream.Read(buffer)); + Assert.Equal(data1, buffer); + + stream.Seek(position2, SeekOrigin.Begin); + Assert.Equal(buffer.Length, stream.Read(buffer)); + Assert.Equal(data2, buffer); + } + } } public class BufferedStream_TestLeaveOpen : TestLeaveOpen From 816653ee950c267c44f1124f573ff567cb22fee6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 2 Jan 2022 20:16:28 -0600 Subject: [PATCH 03/39] [release/6.0] Use Ubuntu 18.04 and dotnet-buildtools-prereq docker images for enterprise linux pipeline (#63014) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Use Ubuntu 18.04 1ES pools and dotnet-buildtools-prereq docker images for enterprise linux pipeline Ubuntu 16.04 is no longer available on Azure Pipelines, move to 18.04 on 1ES pool and the Docker images from dotnet-buildtools-prereq. * Workaround https://github.com/dotnet/runtime/issues/34649 * Use servicing 1ES pool * Fix property name for turning off PGO data It was changed in main with https://github.com/dotnet/runtime/commit/4682098d3a435d3f88942ee9e5c6fd426555963a * Fix property value Co-authored-by: Alexander Köplinger --- eng/pipelines/libraries/enterprise/linux.yml | 5 ++-- .../setup/linuxclient/Dockerfile | 23 ++----------------- 2 files changed, 5 insertions(+), 23 deletions(-) diff --git a/eng/pipelines/libraries/enterprise/linux.yml b/eng/pipelines/libraries/enterprise/linux.yml index 754b533564e3f4..904dc4c87c73cb 100644 --- a/eng/pipelines/libraries/enterprise/linux.yml +++ b/eng/pipelines/libraries/enterprise/linux.yml @@ -19,7 +19,8 @@ pr: - src/libraries/System.Net.Security/* pool: - vmImage: 'ubuntu-16.04' + name: NetCore1ESPool-Svc-Public + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open variables: - template: ../variables.yml @@ -50,7 +51,7 @@ steps: displayName: Test linuxclient connection to web server - bash: | - docker exec linuxclient bash -c '/repo/build.sh -subset clr+libs -runtimeconfiguration release -ci' + docker exec linuxclient bash -c '/repo/build.sh -subset clr+libs -runtimeconfiguration release -ci /p:NoPgoOptimize=true' displayName: Build product sources - bash: | diff --git a/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/linuxclient/Dockerfile b/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/linuxclient/Dockerfile index 8bb94ba8aa9924..853582e7bcf3c1 100644 --- a/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/linuxclient/Dockerfile +++ b/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/linuxclient/Dockerfile @@ -1,31 +1,12 @@ -# Switch to mcr.microsoft.com/dotnet-buildtools/prereqs ubuntu 18.04 images when they are fixed -FROM ubuntu:18.04 +FROM mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-20211022152710-047508b # Prevents dialog prompting when installing packages ARG DEBIAN_FRONTEND=noninteractive -# This 'RUN' step can be removed once dotnet-buildtools/prereqs image is fixed -# -# Makes the image capable of building and running tests in dotnet-runtime repo. -# Add retries to apt-get since the ubuntu package servers have had errors lately such as: -# "E: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/p/publicsuffix/publicsuffix_20180223.1310-1_all.deb Undetermined Error [IP: 91.189.88.31 80]" -ARG APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 -RUN echo "APT::Acquire::Retries \"10\";" > /etc/apt/apt.conf.d/80-retries && \ - apt-get update && \ - apt-get install -y --no-install-recommends apt-transport-https ca-certificates gnupg software-properties-common wget && \ - wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - && \ - apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key 2>/dev/null | apt-key add - && \ - apt-add-repository 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main' && \ - apt-get update && \ - apt-get install -y --no-install-recommends cmake llvm-9 clang-9 lldb-6.0 liblldb-6.0-dev libunwind8 libunwind-dev gettext libicu-dev liblttng-ust-dev libssl-dev libnuma-dev libkrb5-dev locales && \ - locale-gen en_US.UTF-8 && \ - update-locale LANG=en_US.UTF-8 - # Install Kerberos, NTLM, and diagnostic tools COPY ./common/krb5.conf /etc/krb5.conf RUN apt-get update && \ - apt-get install -y --no-install-recommends krb5-user gss-ntlmssp iputils-ping dnsutils nano curl + apt-get install -y --no-install-recommends krb5-user gss-ntlmssp iputils-ping dnsutils nano # Set environment variable to turn on enterprise tests ENV DOTNET_RUNTIME_ENTERPRISETESTS_ENABLED 1 From 2282b00b4bb9754088da92f45e5ac7bda16b58ec Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Sun, 2 Jan 2022 21:18:14 -0500 Subject: [PATCH 04/39] [release/6.0] Add runtime-community.yml pipeline and add s390x job (#62808) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add runtime-community.yml pipeline and add s390x job (#60255) (cherry picked from commit 5f3062d81a5caa444308937539ad136b72218c29) * Don't include s390x in coreclr outerloop runs (#60351) This was a copy-paste mistake, see https://github.com/dotnet/runtime/pull/60255#issuecomment-942539639 (cherry picked from commit 49cf05cdea3baa22592d08d74ce2f0e2c32786a7) * Enable for release/*.* branch, not just main * Fix condition for dependOnEvaluatePaths for runtime-community pipeline (#60552) It was missed where we set a default value for that variable. (cherry picked from commit 157a8b5f72f69e18fbd5ec1f549954e3a28f8f71) * [mono] For cross builds targeting s390x use proper objcopy (#57566) Co-authored-by: Stefan Schulze Frielinghaus (cherry picked from commit 616ace39bb8d9431097b433c31caca6dd934d349) * Use live-built corehost on s390x for creating testhost (#58952) This allows consuming the dotnet host that was just built instead of relying on a downloaded one. We need to move the `host` subset build before the `libs` subset so that the artifacts are available before externals.csproj runs. (cherry picked from commit cab3a63fe3b9810b801ea6a7d9fb958d1af5e380) Co-authored-by: Alexander Köplinger Co-authored-by: stefan-sf-ibm <73470131+stefan-sf-ibm@users.noreply.github.com> --- Directory.Build.props | 4 ++ eng/Subsets.props | 49 +++++++-------- eng/pipelines/common/platform-matrix.yml | 26 ++++++++ eng/pipelines/common/variables.yml | 5 +- eng/pipelines/common/xplat-setup.yml | 2 +- .../libraries/helix-queues-setup.yml | 4 ++ eng/pipelines/runtime-community.yml | 60 +++++++++++++++++++ src/installer/pkg/Directory.Build.props | 1 - src/libraries/externals.csproj | 19 +++++- src/libraries/tests.proj | 4 ++ src/mono/mono.proj | 1 + 11 files changed, 145 insertions(+), 30 deletions(-) create mode 100644 eng/pipelines/runtime-community.yml diff --git a/Directory.Build.props b/Directory.Build.props index 5cb87bb7c25725..54ae373485c0fa 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -210,6 +210,10 @@ $([MSBuild]::NormalizeDirectory('$(MicrosoftNetCoreAppRuntimePackRidDir)', 'native')) + + $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', '$(OutputRid).$(Configuration)', 'corehost')) + + true diff --git a/eng/Subsets.props b/eng/Subsets.props index b2292ac98e61c8..be4c6a8862480b 100644 --- a/eng/Subsets.props +++ b/eng/Subsets.props @@ -120,7 +120,14 @@ - + + + + + + + + @@ -130,13 +137,6 @@ - - - - - - - @@ -263,6 +263,22 @@ + + + + + + + + + + + + + + + + @@ -288,22 +304,7 @@ - - - - - - - - - - - - - - - - + diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index 3beb80193a725a..fbc12a81b5abc6 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -205,6 +205,32 @@ jobs: ${{ insert }}: ${{ parameters.jobParameters }} buildingOnSourceBuildImage: true +# Linux s390x + +- ${{ if containsValue(parameters.platforms, 'Linux_s390x') }}: + - template: xplat-setup.yml + parameters: + jobTemplate: ${{ parameters.jobTemplate }} + helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} + variables: ${{ parameters.variables }} + osGroup: Linux + archType: s390x + targetRid: linux-s390x + platform: Linux_s390x + container: + image: ubuntu-18.04-cross-s390x-20201102145728-d6e0352 + registry: mcr + jobParameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + stagedBuild: ${{ parameters.stagedBuild }} + buildConfig: ${{ parameters.buildConfig }} + ${{ if eq(parameters.passPlatforms, true) }}: + platforms: ${{ parameters.platforms }} + helixQueueGroup: ${{ parameters.helixQueueGroup }} + crossBuild: true + crossrootfsDir: '/crossrootfs/s390x' + ${{ insert }}: ${{ parameters.jobParameters }} + # WebAssembly - ${{ if containsValue(parameters.platforms, 'Browser_wasm') }}: diff --git a/eng/pipelines/common/variables.yml b/eng/pipelines/common/variables.yml index c724f4beb367f9..e2ed01f836cf55 100644 --- a/eng/pipelines/common/variables.yml +++ b/eng/pipelines/common/variables.yml @@ -14,9 +14,10 @@ variables: - name: isFullMatrix value: ${{ and(eq(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }} -# We only run evaluate paths on runtime and runtime-staging pipelines on PRs +# We only run evaluate paths on runtime, runtime-staging and runtime-community pipelines on PRs +# keep in sync with /eng/pipelines/common/xplat-setup.yml - name: dependOnEvaluatePaths - value: ${{ and(eq(variables['Build.Reason'], 'PullRequest'), in(variables['Build.DefinitionName'], 'runtime', 'runtime-staging')) }} + value: ${{ and(eq(variables['Build.Reason'], 'PullRequest'), in(variables['Build.DefinitionName'], 'runtime', 'runtime-staging', 'runtime-community')) }} - name: debugOnPrReleaseOnRolling ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: value: Release diff --git a/eng/pipelines/common/xplat-setup.yml b/eng/pipelines/common/xplat-setup.yml index 2010c653f89cc2..9c9861e3a963f1 100644 --- a/eng/pipelines/common/xplat-setup.yml +++ b/eng/pipelines/common/xplat-setup.yml @@ -22,7 +22,7 @@ jobs: shouldContinueOnError: ${{ and(endsWith(variables['Build.DefinitionName'], 'staging'), eq(variables['Build.Reason'], 'PullRequest')) }} # keep in sync with /eng/pipelines/common/variables.yml - dependOnEvaluatePaths: ${{ and(eq(variables['Build.Reason'], 'PullRequest'), in(variables['Build.DefinitionName'], 'runtime', 'runtime-staging')) }} + dependOnEvaluatePaths: ${{ and(eq(variables['Build.Reason'], 'PullRequest'), in(variables['Build.DefinitionName'], 'runtime', 'runtime-staging', 'runtime-community')) }} variables: # Disable component governance in our CI builds. These builds are not shipping nor diff --git a/eng/pipelines/libraries/helix-queues-setup.yml b/eng/pipelines/libraries/helix-queues-setup.yml index 5a82951839ff4a..73432c9c408b1a 100644 --- a/eng/pipelines/libraries/helix-queues-setup.yml +++ b/eng/pipelines/libraries/helix-queues-setup.yml @@ -82,6 +82,10 @@ jobs: # Limiting interp runs as we don't need as much coverage. - (Debian.10.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-amd64-20210304164434-56c6673 + # Linux s390x + - ${{ if eq(parameters.platform, 'Linux_s390x') }}: + - Ubuntu.2004.S390X.Experimental.Open + # OSX arm64 - ${{ if eq(parameters.platform, 'OSX_arm64') }}: - OSX.1100.ARM64.Open diff --git a/eng/pipelines/runtime-community.yml b/eng/pipelines/runtime-community.yml new file mode 100644 index 00000000000000..9ecb1f2677ae2d --- /dev/null +++ b/eng/pipelines/runtime-community.yml @@ -0,0 +1,60 @@ +trigger: none + +schedules: + - cron: "0 7,19 * * *" # run at 7:00 and 19:00 (UTC) which is 23:00 and 11:00 (PST). + displayName: Runtime-community default schedule + branches: + include: + - main + - release/*.* + always: false # run only if there were changes since the last successful scheduled run. + +variables: + - template: /eng/pipelines/common/variables.yml + +jobs: +# +# Evaluate paths +# +- ${{ if eq(variables.dependOnEvaluatePaths, true) }}: + - template: /eng/pipelines/common/evaluate-default-paths.yml + +# +# s390x +# Build the whole product using Mono and run libraries tests +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: Release + runtimeFlavor: mono + platforms: + - Linux_s390x + variables: + # map dependencies variables to local variables + - name: librariesContainsChange + value: $[ dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'] ] + - name: monoContainsChange + value: $[ dependencies.evaluate_paths.outputs['SetPathVars_mono.containsChange'] ] + jobParameters: + testGroup: innerloop + nameSuffix: AllSubsets_Mono + buildArgs: -s mono+libs+host+packs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true + timeoutInMinutes: 180 + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_mono.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), + eq(variables['isFullMatrix'], true)) + # extra steps, run tests + extraStepsTemplate: /eng/pipelines/libraries/helix.yml + extraStepsParameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_$(_BuildConfig) + condition: >- + or( + eq(variables['librariesContainsChange'], true), + eq(variables['monoContainsChange'], true), + eq(variables['isFullMatrix'], true)) diff --git a/src/installer/pkg/Directory.Build.props b/src/installer/pkg/Directory.Build.props index 10f36b399c73b6..f51718676368a2 100644 --- a/src/installer/pkg/Directory.Build.props +++ b/src/installer/pkg/Directory.Build.props @@ -3,7 +3,6 @@ $(TargetArchitecture) - $(ArtifactsBinDir)$(OutputRid).$(Configuration)\corehost diff --git a/src/libraries/externals.csproj b/src/libraries/externals.csproj index 6ef3ddc0f19015..969ed9603a2cda 100644 --- a/src/libraries/externals.csproj +++ b/src/libraries/externals.csproj @@ -9,6 +9,7 @@ true false true + true @@ -16,7 +17,7 @@ Version="$(MicrosoftDiaSymReaderNativeVersion)" /> - + - + + + + + + + + + + + + + + diff --git a/src/mono/mono.proj b/src/mono/mono.proj index dd8140fb81e8be..fb98ffc1896906 100644 --- a/src/mono/mono.proj +++ b/src/mono/mono.proj @@ -506,6 +506,7 @@ <_Objcopy>objcopy <_Objcopy Condition="'$(Platform)' == 'arm'">arm-linux-$(_LinuxAbi)eabi$(_LinuxFloatAbi)-$(_Objcopy) <_Objcopy Condition="'$(Platform)' == 'arm64'">aarch64-linux-$(_LinuxAbi)-$(_Objcopy) + <_Objcopy Condition="'$(Platform)' == 's390x'">s390x-linux-$(_LinuxAbi)-$(_Objcopy) <_Objcopy Condition="'$(Platform)' == 'x64'">x86_64-linux-$(_LinuxAbi)-$(_Objcopy) <_Objcopy Condition="'$(Platform)' == 'x86'">i686-linux-$(_LinuxAbi)-$(_Objcopy) <_Objcopy Condition="'$(TargetsAndroid)' == 'true'">$(ANDROID_NDK_ROOT)/toolchains/llvm/prebuilt/$(MonoToolchainPrebuiltOS)/bin/$(_Objcopy) From 6f1e03f1bfa04e00627173e51790df700438443e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 2 Jan 2022 20:20:00 -0600 Subject: [PATCH 05/39] [release/6.0] Do not promote struct locals with holes (#62738) * Make sure the combined field size matches the struct size * Fix the condition * Update test and include containHoles condition Co-authored-by: Kunal Pathak --- src/coreclr/jit/lclvars.cpp | 9 ++- .../JitBlue/Runtime_62597/Runtime_62597.cs | 70 +++++++++++++++++++ .../Runtime_62597/Runtime_62597.csproj | 13 ++++ 3 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 src/tests/JIT/Regression/JitBlue/Runtime_62597/Runtime_62597.cs create mode 100644 src/tests/JIT/Regression/JitBlue/Runtime_62597/Runtime_62597.csproj diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index df083c7c3a7524..6b7032336bd2d2 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -2103,8 +2103,13 @@ bool Compiler::StructPromotionHelper::ShouldPromoteStructVar(unsigned lclNum) // multiple registers? if (compiler->lvaIsMultiregStruct(varDsc, compiler->info.compIsVarArgs)) { - if ((structPromotionInfo.fieldCnt != 2) && - !((structPromotionInfo.fieldCnt == 1) && varTypeIsSIMD(structPromotionInfo.fields[0].fldType))) + if (structPromotionInfo.containsHoles && structPromotionInfo.customLayout) + { + JITDUMP("Not promoting multi-reg struct local V%02u with holes.\n", lclNum); + shouldPromote = false; + } + else if ((structPromotionInfo.fieldCnt != 2) && + !((structPromotionInfo.fieldCnt == 1) && varTypeIsSIMD(structPromotionInfo.fields[0].fldType))) { JITDUMP("Not promoting multireg struct local V%02u, because lvIsParam is true, #fields != 2 and it's " "not a single SIMD.\n", diff --git a/src/tests/JIT/Regression/JitBlue/Runtime_62597/Runtime_62597.cs b/src/tests/JIT/Regression/JitBlue/Runtime_62597/Runtime_62597.cs new file mode 100644 index 00000000000000..fb3ccef4ade51b --- /dev/null +++ b/src/tests/JIT/Regression/JitBlue/Runtime_62597/Runtime_62597.cs @@ -0,0 +1,70 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// +// Note: In below test case, we were not honoring the fact that the explicit struct size +// of struct is 32 bytes while the only 2 fields it has is just 2 bytes. In such case, +// we would pass partial struct value. +using System; +using System.Reflection.Emit; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; + +[StructLayout(LayoutKind.Explicit, Size = 32)] +public readonly unsafe struct SmallString +{ + [FieldOffset(0)] private readonly byte _length; + [FieldOffset(1)] private readonly byte _firstByte; + + public SmallString(string value) + { + fixed (char* srcPtr = value) + fixed (byte* destPtr = &_firstByte) + { + Encoding.ASCII.GetBytes(srcPtr, value.Length, destPtr, value.Length); + } + + _length = (byte)value.Length; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public byte Dump() + { + fixed (byte* ptr = &_firstByte) + { + byte* next = ptr + 1; + return *next; + } + } +} + +public static class Program +{ + static int result = 0; + public static int Main() + { + var value = new SmallString("foobar"); + + TheTest(value); + + return result; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static void TheTest(SmallString foo) + { + Execute(foo); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static object Execute(SmallString foo) + { + byte value = foo.Dump(); + // 111 corresponds to the ASCII code of 2nd characted of string "foobar" i.e. ASCII value of 'o'. + if (value == 111) + { + result = 100; + } + return new StringBuilder(); + } +} diff --git a/src/tests/JIT/Regression/JitBlue/Runtime_62597/Runtime_62597.csproj b/src/tests/JIT/Regression/JitBlue/Runtime_62597/Runtime_62597.csproj new file mode 100644 index 00000000000000..e822a8b10a5a6f --- /dev/null +++ b/src/tests/JIT/Regression/JitBlue/Runtime_62597/Runtime_62597.csproj @@ -0,0 +1,13 @@ + + + Exe + True + + + None + True + + + + + \ No newline at end of file From 7d82ac75ec9de74c5277edaf55510953ca6ce4e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Mon, 3 Jan 2022 03:26:16 +0100 Subject: [PATCH 06/39] [wasm][release/6.0] Reduce number of calls to setTimer (#62705) * Port changes to the TimerQueue from https://github.com/dotnet/runtime/pull/62433. Test is only manual. Xharness version on this branch triggers timer queue in a way that makes the test useless. Co-authored-by: Pavel Savara * Replace main method with plain static method for running timer tests. * Rename index.html to simple.html to avoid collision with generated index.html for unit tests. Co-authored-by: Pavel Savara --- ...me.InteropServices.JavaScript.Tests.csproj | 8 + .../JavaScript/Simple/SimpleTest.cs | 66 ++++++ .../JavaScript/Simple/TimerTests.cs | 192 ++++++++++++++++++ .../tests/simple.html | 14 ++ .../tests/simple.js | 79 +++++++ .../Threading/TimerQueue.Browser.Mono.cs | 74 +++++-- src/mono/wasm/runtime/library_mono.js | 8 +- 7 files changed, 424 insertions(+), 17 deletions(-) create mode 100644 src/libraries/System.Private.Runtime.InteropServices.JavaScript/tests/System/Runtime/InteropServices/JavaScript/Simple/SimpleTest.cs create mode 100644 src/libraries/System.Private.Runtime.InteropServices.JavaScript/tests/System/Runtime/InteropServices/JavaScript/Simple/TimerTests.cs create mode 100644 src/libraries/System.Private.Runtime.InteropServices.JavaScript/tests/simple.html create mode 100644 src/libraries/System.Private.Runtime.InteropServices.JavaScript/tests/simple.js diff --git a/src/libraries/System.Private.Runtime.InteropServices.JavaScript/tests/System.Private.Runtime.InteropServices.JavaScript.Tests.csproj b/src/libraries/System.Private.Runtime.InteropServices.JavaScript/tests/System.Private.Runtime.InteropServices.JavaScript.Tests.csproj index fe9348f4678320..8c552fccec58c0 100644 --- a/src/libraries/System.Private.Runtime.InteropServices.JavaScript/tests/System.Private.Runtime.InteropServices.JavaScript.Tests.csproj +++ b/src/libraries/System.Private.Runtime.InteropServices.JavaScript/tests/System.Private.Runtime.InteropServices.JavaScript.Tests.csproj @@ -17,6 +17,14 @@ + + + + + + + + diff --git a/src/libraries/System.Private.Runtime.InteropServices.JavaScript/tests/System/Runtime/InteropServices/JavaScript/Simple/SimpleTest.cs b/src/libraries/System.Private.Runtime.InteropServices.JavaScript/tests/System/Runtime/InteropServices/JavaScript/Simple/SimpleTest.cs new file mode 100644 index 00000000000000..18318330ffcf67 --- /dev/null +++ b/src/libraries/System.Private.Runtime.InteropServices.JavaScript/tests/System/Runtime/InteropServices/JavaScript/Simple/SimpleTest.cs @@ -0,0 +1,66 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace System.Runtime.InteropServices.JavaScript.Tests +{ + public static class SimpleTest + { + public static async Task Test() + { + var tests = new List>(); + tests.Add(TimerTests.T0_NoTimer); + tests.Add(TimerTests.T1_OneTimer); + tests.Add(TimerTests.T2_SecondTimerEarlier); + tests.Add(TimerTests.T3_SecondTimerLater); + tests.Add(TimerTests.T5_FiveTimers); + + try + { + Console.WriteLine("SimpleMain start test!"); + var failures = 0; + var failureNames = new List(); + foreach (var test in tests) + { + var failed = await RunTest(test); + if (failed != null) + { + failureNames.Add(failed); + failures++; + } + } + + foreach (var failure in failureNames) + { + Console.WriteLine(failure); + } + Console.WriteLine($"{Environment.NewLine}=== TEST EXECUTION SUMMARY ==={Environment.NewLine}Total: {tests.Count}, Failed: {failures}"); + return failures; + } + catch (Exception ex) + { + Console.WriteLine(ex.ToString()); + return -1; + } + } + + private static async Task RunTest(Func action) + { + try + { + Console.WriteLine("[STRT] " + action.Method.Name); + await action(); + Console.WriteLine("[DONE] " + action.Method.Name); + return null; + } + catch (Exception ex) + { + var message="[FAIL] "+action.Method.Name + " " + ex.Message; + Console.WriteLine(message); + return message; + } + } + } +} diff --git a/src/libraries/System.Private.Runtime.InteropServices.JavaScript/tests/System/Runtime/InteropServices/JavaScript/Simple/TimerTests.cs b/src/libraries/System.Private.Runtime.InteropServices.JavaScript/tests/System/Runtime/InteropServices/JavaScript/Simple/TimerTests.cs new file mode 100644 index 00000000000000..3f7f7ac8add77c --- /dev/null +++ b/src/libraries/System.Private.Runtime.InteropServices.JavaScript/tests/System/Runtime/InteropServices/JavaScript/Simple/TimerTests.cs @@ -0,0 +1,192 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace System.Runtime.InteropServices.JavaScript.Tests +{ + public static class TimerTests + { + static JSObject _timersHelper = (JSObject)Runtime.GetGlobalObject("timersHelper"); + static Function _installWrapper = (Function)_timersHelper.GetObjectProperty("install"); + static Function _getRegisterCount = (Function)_timersHelper.GetObjectProperty("getRegisterCount"); + static Function _getHitCount = (Function)_timersHelper.GetObjectProperty("getHitCount"); + static Function _cleanupWrapper = (Function)_timersHelper.GetObjectProperty("cleanup"); + + static public async Task T0_NoTimer() + { + try + { + _installWrapper.Call(); + + var setCounter = (int)_getRegisterCount.Call(); + Assert.Equal(0, setCounter); + } + finally + { + await WaitForCleanup(); + } + } + + static public async Task T1_OneTimer() + { + int wasCalled = 0; + Timer? timer = null; + try + { + _installWrapper.Call(); + + timer = new Timer((_) => + { + Console.WriteLine("In timer"); + wasCalled++; + }, null, 10, 0); + + var setCounter = (int)_getRegisterCount.Call(); + Assert.True(0 == wasCalled, $"wasCalled: {wasCalled}"); + Assert.True(1 == setCounter, $"setCounter: {setCounter}"); + } + finally + { + await WaitForCleanup(); + Assert.True(1 == wasCalled, $"wasCalled: {wasCalled}"); + timer?.Dispose(); + } + } + + static public async Task T2_SecondTimerEarlier() + { + int wasCalled = 0; + Timer? timer1 = null; + Timer? timer2 = null; + try + { + _installWrapper.Call(); + + timer1 = new Timer((_) => + { + Console.WriteLine("In timer1"); + wasCalled++; + }, null, 10, 0); + timer2 = new Timer((_) => + { + Console.WriteLine("In timer2"); + wasCalled++; + }, null, 5, 0); + + var setCounter = (int)_getRegisterCount.Call(); + Assert.True(2 == setCounter, $"setCounter: {setCounter}"); + Assert.True(0 == wasCalled, $"wasCalled: {wasCalled}"); + + } + finally + { + await WaitForCleanup(); + Assert.True(2 == wasCalled, $"wasCalled: {wasCalled}"); + timer1?.Dispose(); + timer2?.Dispose(); + } + } + + static public async Task T3_SecondTimerLater() + { + int wasCalled = 0; + Timer? timer1 = null; + Timer? timer2 = null; + try + { + _installWrapper.Call(); + + timer1 = new Timer((_) => + { + Console.WriteLine("In timer1"); + wasCalled++; + }, null, 10, 0); + timer2 = new Timer((_) => + { + Console.WriteLine("In timer2"); + wasCalled++; + }, null, 20, 0); + + var setCounter = (int)_getRegisterCount.Call(); + Assert.True(0 == wasCalled, $"wasCalled: {wasCalled}"); + Assert.True(1 == setCounter, $"setCounter: {setCounter}"); + } + finally + { + await WaitForCleanup(); + Assert.True(2 == wasCalled, $"wasCalled: {wasCalled}"); + timer1?.Dispose(); + timer2?.Dispose(); + } + } + + static public async Task T5_FiveTimers() + { + int wasCalled = 0; + Timer? timer1 = null; + Timer? timer2 = null; + Timer? timer3 = null; + Timer? timer4 = null; + Timer? timer5 = null; + try + { + _installWrapper.Call(); + + timer1 = new Timer((_) => + { + Console.WriteLine("In timer1"); + wasCalled++; + }, null, 800, 0); + timer2 = new Timer((_) => + { + Console.WriteLine("In timer2"); + wasCalled++; + }, null, 600, 0); + timer3 = new Timer((_) => + { + Console.WriteLine("In timer3"); + wasCalled++; + }, null, 400, 0); + timer4 = new Timer((_) => + { + Console.WriteLine("In timer4"); + wasCalled++; + }, null, 200, 0); + timer5 = new Timer((_) => + { + Console.WriteLine("In timer5"); + wasCalled++; + }, null, 000, 0); + + var setCounter = (int)_getRegisterCount.Call(); + Assert.True(0 == wasCalled, $"wasCalled: {wasCalled}"); + Assert.True(5 == setCounter, $"setCounter: {setCounter}"); + } + finally + { + await WaitForCleanup(); + var hitCounter = (int)_getHitCount.Call(); + var setCounter = (int)_getRegisterCount.Call(); + Assert.True(5 == wasCalled, $"wasCalled: {wasCalled}"); + Assert.True(8 == hitCounter, $"hitCounter: {hitCounter}"); + Assert.True(12 == setCounter, $"setCounter: {setCounter}"); + timer1?.Dispose(); + timer2?.Dispose(); + timer3?.Dispose(); + timer4?.Dispose(); + timer5?.Dispose(); + } + } + + static private async Task WaitForCleanup() + { + Console.WriteLine("wait for cleanup begin"); + await Task.Delay(1000); + _cleanupWrapper.Call(); + Console.WriteLine("wait for cleanup end"); + } + } +} diff --git a/src/libraries/System.Private.Runtime.InteropServices.JavaScript/tests/simple.html b/src/libraries/System.Private.Runtime.InteropServices.JavaScript/tests/simple.html new file mode 100644 index 00000000000000..59acbcfcd1cd1e --- /dev/null +++ b/src/libraries/System.Private.Runtime.InteropServices.JavaScript/tests/simple.html @@ -0,0 +1,14 @@ + + + + + + TESTS + + + + + + + + \ No newline at end of file diff --git a/src/libraries/System.Private.Runtime.InteropServices.JavaScript/tests/simple.js b/src/libraries/System.Private.Runtime.InteropServices.JavaScript/tests/simple.js new file mode 100644 index 00000000000000..bd9f900aafebf6 --- /dev/null +++ b/src/libraries/System.Private.Runtime.InteropServices.JavaScript/tests/simple.js @@ -0,0 +1,79 @@ +class TimersHelper { + install() { + const measuredCallbackName = "mono_wasm_set_timeout_exec"; + globalThis.registerCounter = 0; + globalThis.hitCounter = 0; + console.log("install") + if (!globalThis.originalSetTimeout) { + globalThis.originalSetTimeout = globalThis.setTimeout; + } + globalThis.setTimeout = (cb, time) => { + var start = Date.now().valueOf(); + if (cb.name === measuredCallbackName) { + globalThis.registerCounter++; + console.log(`registerCounter: ${globalThis.registerCounter} now:${start} delay:${time}`) + } + return globalThis.originalSetTimeout(() => { + if (cb.name === measuredCallbackName) { + var hit = Date.now().valueOf(); + globalThis.hitCounter++; + var delta = hit - start; + console.log(`hitCounter: ${globalThis.hitCounter} now:${hit} delay:${time} delta:${delta}`) + } + cb(); + }, time); + }; + } + + getRegisterCount() { + console.log(`registerCounter: ${globalThis.registerCounter} `) + return globalThis.registerCounter; + } + + getHitCount() { + console.log(`hitCounter: ${globalThis.hitCounter} `) + return globalThis.hitCounter; + } + + cleanup() { + console.log(`cleanup registerCounter: ${globalThis.registerCounter} hitCounter: ${globalThis.hitCounter} `) + globalThis.setTimeout = globalThis.originalSetTimeout; + } +} + +globalThis.timersHelper = new TimersHelper(); + +var Module = { + + config: null, + + preInit: async function() { + await MONO.mono_wasm_load_config("./mono-config.json"); // sets Module.config implicitly + }, + + // Called when the runtime is initialized and wasm is ready + onRuntimeInitialized: function () { + if (!Module.config || Module.config.error) { + console.log("No config found"); + return; + } + + Module.config.loaded_cb = function () { + try { + BINDING.call_static_method("[System.Private.Runtime.InteropServices.JavaScript.Tests] System.Runtime.InteropServices.JavaScript.Tests.SimpleTest:Test", []); + } catch (error) { + throw (error); + } + }; + Module.config.fetch_file_cb = function (asset) { + return fetch (asset, { credentials: 'same-origin' }); + } + + try + { + MONO.mono_load_runtime_and_bcl_args (Module.config); + } catch (error) { + throw(error); + } + }, +}; diff --git a/src/mono/System.Private.CoreLib/src/System/Threading/TimerQueue.Browser.Mono.cs b/src/mono/System.Private.CoreLib/src/System/Threading/TimerQueue.Browser.Mono.cs index 5ad4ea2e23cead..4187cfcb829089 100644 --- a/src/mono/System.Private.CoreLib/src/System/Threading/TimerQueue.Browser.Mono.cs +++ b/src/mono/System.Private.CoreLib/src/System/Threading/TimerQueue.Browser.Mono.cs @@ -17,7 +17,9 @@ internal partial class TimerQueue { private static List? s_scheduledTimers; private static List? s_scheduledTimersToFire; + private static long s_shortestDueTimeMs = long.MaxValue; + // this means that it's in the s_scheduledTimers collection, not that it's the one which would run on the next TimeoutCallback private bool _isScheduled; private long _scheduledDueTimeMs; @@ -27,24 +29,25 @@ private TimerQueue(int id) [DynamicDependency("TimeoutCallback")] // The id argument is unused in netcore + // This replaces the current pending setTimeout with shorter one [MethodImplAttribute(MethodImplOptions.InternalCall)] private static extern void SetTimeout(int timeout, int id); // Called by mini-wasm.c:mono_set_timeout_exec private static void TimeoutCallback() { - int shortestWaitDurationMs = PumpTimerQueue(); + // always only have one scheduled at a time + s_shortestDueTimeMs = long.MaxValue; - if (shortestWaitDurationMs != int.MaxValue) - { - SetTimeout((int)shortestWaitDurationMs, 0); - } + long currentTimeMs = TickCount64; + ReplaceNextSetTimeout(PumpTimerQueue(currentTimeMs), currentTimeMs); } + // this is called with shortest of timers scheduled on the particular TimerQueue private bool SetTimer(uint actualDuration) { Debug.Assert((int)actualDuration >= 0); - long dueTimeMs = TickCount64 + (int)actualDuration; + long currentTimeMs = TickCount64; if (!_isScheduled) { s_scheduledTimers ??= new List(Instances.Length); @@ -52,24 +55,65 @@ private bool SetTimer(uint actualDuration) s_scheduledTimers.Add(this); _isScheduled = true; } - _scheduledDueTimeMs = dueTimeMs; - SetTimeout((int)actualDuration, 0); + + _scheduledDueTimeMs = currentTimeMs + (int)actualDuration; + + ReplaceNextSetTimeout(ShortestDueTime(), currentTimeMs); return true; } - private static int PumpTimerQueue() + // shortest time of all TimerQueues + private static void ReplaceNextSetTimeout(long shortestDueTimeMs, long currentTimeMs) { - if (s_scheduledTimersToFire == null) + if (shortestDueTimeMs == int.MaxValue) + { + return; + } + + // this also covers s_shortestDueTimeMs = long.MaxValue when none is scheduled + if (s_shortestDueTimeMs > shortestDueTimeMs) + { + s_shortestDueTimeMs = shortestDueTimeMs; + int shortestWait = Math.Max((int)(shortestDueTimeMs - currentTimeMs), 0); + // this would cancel the previous schedule and create shorter one + // it is expensive call + SetTimeout(shortestWait, 0); + } + } + + private static long ShortestDueTime() + { + if (s_scheduledTimers == null) { return int.MaxValue; } + long shortestDueTimeMs = long.MaxValue; + var timers = s_scheduledTimers!; + for (int i = timers.Count - 1; i >= 0; --i) + { + TimerQueue timer = timers[i]; + if (timer._scheduledDueTimeMs < shortestDueTimeMs) + { + shortestDueTimeMs = timer._scheduledDueTimeMs; + } + } + + return shortestDueTimeMs; + } + + private static long PumpTimerQueue(long currentTimeMs) + { + if (s_scheduledTimersToFire == null) + { + return ShortestDueTime(); + } + List timersToFire = s_scheduledTimersToFire!; List timers; timers = s_scheduledTimers!; - long currentTimeMs = TickCount64; - int shortestWaitDurationMs = int.MaxValue; + long shortestDueTimeMs = int.MaxValue; for (int i = timers.Count - 1; i >= 0; --i) { TimerQueue timer = timers[i]; @@ -88,9 +132,9 @@ private static int PumpTimerQueue() continue; } - if (waitDurationMs < shortestWaitDurationMs) + if (timer._scheduledDueTimeMs < shortestDueTimeMs) { - shortestWaitDurationMs = (int)waitDurationMs; + shortestDueTimeMs = timer._scheduledDueTimeMs; } } @@ -103,7 +147,7 @@ private static int PumpTimerQueue() timersToFire.Clear(); } - return shortestWaitDurationMs; + return shortestDueTimeMs; } } } diff --git a/src/mono/wasm/runtime/library_mono.js b/src/mono/wasm/runtime/library_mono.js index 8f319841d024ea..6c5dbe5e0d0d45 100644 --- a/src/mono/wasm/runtime/library_mono.js +++ b/src/mono/wasm/runtime/library_mono.js @@ -1506,8 +1506,12 @@ var MonoSupportLib = { mono_set_timeout: function (timeout, id) { if (typeof globalThis.setTimeout === 'function') { - globalThis.setTimeout (function () { - MONO.mono_wasm_set_timeout_exec (id); + if (MONO.lastScheduleTimeoutId) { + globalThis.clearTimeout(MONO.lastScheduleTimeoutId); + MONO.lastScheduleTimeoutId = undefined; + } + MONO.lastScheduleTimeoutId = globalThis.setTimeout(function mono_wasm_set_timeout_exec () { + MONO.mono_wasm_set_timeout_exec(id); }, timeout); } else { ++MONO.pump_count; From bdd6647c70932d093c7a7fda3c9232d2664f62fa Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 3 Jan 2022 10:32:20 -0600 Subject: [PATCH 07/39] fix issue where HTTP2 connection could be scavenged but not disposed (#62236) Co-authored-by: Geoffrey Kizer --- .../System/Net/Http/SocketsHttpHandler/HttpConnectionPool.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPool.cs b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPool.cs index 75b83fbe8bae61..c51adc80d5c301 100644 --- a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPool.cs +++ b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPool.cs @@ -2004,7 +2004,8 @@ static int ScavengeConnectionList(List list, ref List? if (freeIndex < list.Count) { // We know the connection at freeIndex is unusable, so dispose of it. - toDispose ??= new List { list[freeIndex] }; + toDispose ??= new List(); + toDispose.Add(list[freeIndex]); // Find the first item after the one to be removed that should be kept. int current = freeIndex + 1; From 45d1f9ecebb52fbca6f9b6fd0ed6f0ab50f09ad7 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 3 Jan 2022 08:53:52 -0800 Subject: [PATCH 08/39] Update dependencies from https://github.com/dotnet/arcade build 20211214.2 (#63007) Microsoft.DotNet.XUnitExtensions , Microsoft.DotNet.VersionTools.Tasks , Microsoft.DotNet.Build.Tasks.Workloads , Microsoft.DotNet.Build.Tasks.Templating , Microsoft.DotNet.Build.Tasks.TargetFramework.Sdk , Microsoft.DotNet.Build.Tasks.Packaging , Microsoft.DotNet.Build.Tasks.Installers , Microsoft.DotNet.Build.Tasks.Feed , Microsoft.DotNet.Build.Tasks.Archives , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.ApiCompat , Microsoft.DotNet.CodeAnalysis , Microsoft.DotNet.XUnitConsoleRunner , Microsoft.DotNet.GenFacades , Microsoft.DotNet.GenAPI , Microsoft.DotNet.RemoteExecutor , Microsoft.DotNet.PackageTesting , Microsoft.DotNet.Helix.Sdk , Microsoft.DotNet.SharedFramework.Sdk From Version 6.0.0-beta.21609.4 -> To Version 6.0.0-beta.21614.2 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 76 ++++++++++++++++++++--------------------- eng/Versions.props | 30 ++++++++-------- global.json | 8 ++--- 3 files changed, 57 insertions(+), 57 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 5f06553dfe751e..cee170f2de925a 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -18,77 +18,77 @@ - + https://github.com/dotnet/arcade - 7421b55f46aff8373764016d942b23cbf87c75cb + 47f8ea1d7ef3efd5d4fa93ccb79ccccf4182095e - + https://github.com/dotnet/arcade - 7421b55f46aff8373764016d942b23cbf87c75cb + 47f8ea1d7ef3efd5d4fa93ccb79ccccf4182095e - + https://github.com/dotnet/arcade - 7421b55f46aff8373764016d942b23cbf87c75cb + 47f8ea1d7ef3efd5d4fa93ccb79ccccf4182095e - + https://github.com/dotnet/arcade - 7421b55f46aff8373764016d942b23cbf87c75cb + 47f8ea1d7ef3efd5d4fa93ccb79ccccf4182095e - + https://github.com/dotnet/arcade - 7421b55f46aff8373764016d942b23cbf87c75cb + 47f8ea1d7ef3efd5d4fa93ccb79ccccf4182095e - + https://github.com/dotnet/arcade - 7421b55f46aff8373764016d942b23cbf87c75cb + 47f8ea1d7ef3efd5d4fa93ccb79ccccf4182095e - + https://github.com/dotnet/arcade - 7421b55f46aff8373764016d942b23cbf87c75cb + 47f8ea1d7ef3efd5d4fa93ccb79ccccf4182095e - + https://github.com/dotnet/arcade - 7421b55f46aff8373764016d942b23cbf87c75cb + 47f8ea1d7ef3efd5d4fa93ccb79ccccf4182095e - + https://github.com/dotnet/arcade - 7421b55f46aff8373764016d942b23cbf87c75cb + 47f8ea1d7ef3efd5d4fa93ccb79ccccf4182095e - + https://github.com/dotnet/arcade - 7421b55f46aff8373764016d942b23cbf87c75cb + 47f8ea1d7ef3efd5d4fa93ccb79ccccf4182095e - + https://github.com/dotnet/arcade - 7421b55f46aff8373764016d942b23cbf87c75cb + 47f8ea1d7ef3efd5d4fa93ccb79ccccf4182095e - + https://github.com/dotnet/arcade - 7421b55f46aff8373764016d942b23cbf87c75cb + 47f8ea1d7ef3efd5d4fa93ccb79ccccf4182095e - + https://github.com/dotnet/arcade - 7421b55f46aff8373764016d942b23cbf87c75cb + 47f8ea1d7ef3efd5d4fa93ccb79ccccf4182095e - + https://github.com/dotnet/arcade - 7421b55f46aff8373764016d942b23cbf87c75cb + 47f8ea1d7ef3efd5d4fa93ccb79ccccf4182095e - + https://github.com/dotnet/arcade - 7421b55f46aff8373764016d942b23cbf87c75cb + 47f8ea1d7ef3efd5d4fa93ccb79ccccf4182095e - + https://github.com/dotnet/arcade - 7421b55f46aff8373764016d942b23cbf87c75cb + 47f8ea1d7ef3efd5d4fa93ccb79ccccf4182095e - + https://github.com/dotnet/arcade - 7421b55f46aff8373764016d942b23cbf87c75cb + 47f8ea1d7ef3efd5d4fa93ccb79ccccf4182095e - + https://github.com/dotnet/arcade - 7421b55f46aff8373764016d942b23cbf87c75cb + 47f8ea1d7ef3efd5d4fa93ccb79ccccf4182095e https://github.com/microsoft/vstest @@ -210,9 +210,9 @@ https://github.com/dotnet/xharness e9669dc84ecd668d3bbb748758103e23b394ffef - + https://github.com/dotnet/arcade - 7421b55f46aff8373764016d942b23cbf87c75cb + 47f8ea1d7ef3efd5d4fa93ccb79ccccf4182095e https://dev.azure.com/dnceng/internal/_git/dotnet-optimization diff --git a/eng/Versions.props b/eng/Versions.props index 7c86ef619df422..afc7e5a00f55ad 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -55,21 +55,21 @@ 1.0.0-rc.2.21511.46 - 6.0.0-beta.21609.4 - 6.0.0-beta.21609.4 - 6.0.0-beta.21609.4 - 6.0.0-beta.21609.4 - 6.0.0-beta.21609.4 - 6.0.0-beta.21609.4 - 2.5.1-beta.21609.4 - 6.0.0-beta.21609.4 - 6.0.0-beta.21609.4 - 6.0.0-beta.21609.4 - 6.0.0-beta.21609.4 - 6.0.0-beta.21609.4 - 6.0.0-beta.21609.4 - 6.0.0-beta.21609.4 - 6.0.0-beta.21609.4 + 6.0.0-beta.21614.2 + 6.0.0-beta.21614.2 + 6.0.0-beta.21614.2 + 6.0.0-beta.21614.2 + 6.0.0-beta.21614.2 + 6.0.0-beta.21614.2 + 2.5.1-beta.21614.2 + 6.0.0-beta.21614.2 + 6.0.0-beta.21614.2 + 6.0.0-beta.21614.2 + 6.0.0-beta.21614.2 + 6.0.0-beta.21614.2 + 6.0.0-beta.21614.2 + 6.0.0-beta.21614.2 + 6.0.0-beta.21614.2 6.0.0-preview.1.102 diff --git a/global.json b/global.json index a8cba13f4f7149..4d822f4ffa563c 100644 --- a/global.json +++ b/global.json @@ -12,10 +12,10 @@ "python3": "3.7.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Build.Tasks.TargetFramework.Sdk": "6.0.0-beta.21609.4", - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21609.4", - "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21609.4", - "Microsoft.DotNet.SharedFramework.Sdk": "6.0.0-beta.21609.4", + "Microsoft.DotNet.Build.Tasks.TargetFramework.Sdk": "6.0.0-beta.21614.2", + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21614.2", + "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21614.2", + "Microsoft.DotNet.SharedFramework.Sdk": "6.0.0-beta.21614.2", "Microsoft.Build.NoTargets": "3.1.0", "Microsoft.Build.Traversal": "3.0.23", "Microsoft.NET.Sdk.IL": "6.0.0-rc.1.21415.6" From f86caa54678535ead8e1977da37025a96e2afe8a Mon Sep 17 00:00:00 2001 From: Jan Vorlicek Date: Tue, 4 Jan 2022 23:51:52 +0100 Subject: [PATCH 09/39] [release/6.0] Fix build with Clang 13 (#63314) * Fix clang 13 induced runtime issues (#62170) The clang 13 optimizer started to assume that "this" pointer is always properly aligned. That lead to elimination of some code that was actually needed. It also takes pointer aliasing rules more strictly in one place in jit. That caused the optimizer to falsely assume that a callee with an argument passed by reference is not modifying that argument and used a stale copy of the original value at the caller site. This change fixes both of the issues. With this fix, runtime compiled using clang 13 seems to be fully functional. * Fix build with clang 13 (#60328) --- eng/native/configurecompiler.cmake | 11 ++++++---- src/coreclr/inc/corhlpr.h | 8 ++++---- src/coreclr/jit/bitsetasshortlong.h | 4 ++-- src/coreclr/jit/inlinepolicy.h | 2 +- src/libraries/Native/Unix/CMakeLists.txt | 3 +++ .../Native/Unix/System.Native/pal_process.c | 20 ++++++++++++++++++- 6 files changed, 36 insertions(+), 12 deletions(-) diff --git a/eng/native/configurecompiler.cmake b/eng/native/configurecompiler.cmake index c22469f567b726..502d432f17bb3f 100644 --- a/eng/native/configurecompiler.cmake +++ b/eng/native/configurecompiler.cmake @@ -340,15 +340,17 @@ if (CLR_CMAKE_HOST_UNIX) #These seem to indicate real issues add_compile_options($<$:-Wno-invalid-offsetof>) - if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") + add_compile_options(-Wno-unused-but-set-variable) + + if (CMAKE_C_COMPILER_ID MATCHES "Clang") + add_compile_options(-Wno-unknown-warning-option) + # The -ferror-limit is helpful during the porting, it makes sure the compiler doesn't stop # after hitting just about 20 errors. add_compile_options(-ferror-limit=4096) # Disabled warnings add_compile_options(-Wno-unused-private-field) - # Explicit constructor calls are not supported by clang (this->ClassName::ClassName()) - add_compile_options(-Wno-microsoft) # There are constants of type BOOL used in a condition. But BOOL is defined as int # and so the compiler thinks that there is a mistake. add_compile_options(-Wno-constant-logical-operand) @@ -363,8 +365,9 @@ if (CLR_CMAKE_HOST_UNIX) # to a struct or a class that has virtual members or a base class. In that case, clang # may not generate the same object layout as MSVC. add_compile_options(-Wno-incompatible-ms-struct) + + add_compile_options(-Wno-reserved-identifier) else() - add_compile_options(-Wno-unused-but-set-variable) add_compile_options(-Wno-unknown-pragmas) add_compile_options(-Wno-uninitialized) add_compile_options(-Wno-strict-aliasing) diff --git a/src/coreclr/inc/corhlpr.h b/src/coreclr/inc/corhlpr.h index 450514da95c180..427e8cdc0ff5c5 100644 --- a/src/coreclr/inc/corhlpr.h +++ b/src/coreclr/inc/corhlpr.h @@ -336,7 +336,7 @@ struct COR_ILMETHOD_SECT const COR_ILMETHOD_SECT* Next() const { if (!More()) return(0); - return ((COR_ILMETHOD_SECT*)(((BYTE *)this) + DataSize()))->Align(); + return ((COR_ILMETHOD_SECT*)Align(((BYTE *)this) + DataSize())); } const BYTE* Data() const @@ -374,9 +374,9 @@ struct COR_ILMETHOD_SECT return((AsSmall()->Kind & CorILMethod_Sect_FatFormat) != 0); } - const COR_ILMETHOD_SECT* Align() const + static const void* Align(const void* p) { - return((COR_ILMETHOD_SECT*) ((((UINT_PTR) this) + 3) & ~3)); + return((void*) ((((UINT_PTR) p) + 3) & ~3)); } protected: @@ -579,7 +579,7 @@ typedef struct tagCOR_ILMETHOD_FAT : IMAGE_COR_ILMETHOD_FAT const COR_ILMETHOD_SECT* GetSect() const { if (!More()) return (0); - return(((COR_ILMETHOD_SECT*) (GetCode() + GetCodeSize()))->Align()); + return(((COR_ILMETHOD_SECT*) COR_ILMETHOD_SECT::Align(GetCode() + GetCodeSize()))); } } COR_ILMETHOD_FAT; diff --git a/src/coreclr/jit/bitsetasshortlong.h b/src/coreclr/jit/bitsetasshortlong.h index dce54d6a5ca3ab..365cf346a10ac2 100644 --- a/src/coreclr/jit/bitsetasshortlong.h +++ b/src/coreclr/jit/bitsetasshortlong.h @@ -345,7 +345,7 @@ class BitSetOpssa_flags) & SA_SIGINFO) + { + // work around -Wcast-function-type + void (*tmp)(void) = (void (*)(void))sa->sa_sigaction; + return (void (*)(int))tmp; + } + else + { + return sa->sa_handler; + } +} + int32_t SystemNative_ForkAndExecProcess(const char* filename, char* const argv[], char* const envp[], @@ -371,7 +389,7 @@ int32_t SystemNative_ForkAndExecProcess(const char* filename, } if (!sigaction(sig, NULL, &sa_old)) { - void (*oldhandler)(int) = (((unsigned int)sa_old.sa_flags) & SA_SIGINFO) ? (void (*)(int))sa_old.sa_sigaction : sa_old.sa_handler; + void (*oldhandler)(int) = handler_from_sigaction (&sa_old); if (oldhandler != SIG_IGN && oldhandler != SIG_DFL) { // It has a custom handler, put the default handler back. From 51b6749b109dbac41e832d1f8f5c9d99ac0a6765 Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Wed, 5 Jan 2022 13:08:12 -0600 Subject: [PATCH 10/39] [release/6.0] bump sourcebuild leg timeout (#63298) --- eng/pipelines/runtime-official.yml | 1 + src/installer/tests/TestUtils/Command.cs | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 906798926b8389..217755efa13a7b 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -336,6 +336,7 @@ stages: extraStepsTemplate: /eng/pipelines/common/upload-intermediate-artifacts-step.yml extraStepsParameters: name: SourceBuildPackages + timeoutInMinutes: 95 # diff --git a/src/installer/tests/TestUtils/Command.cs b/src/installer/tests/TestUtils/Command.cs index 3c3e5607dde90a..91f85be3584324 100644 --- a/src/installer/tests/TestUtils/Command.cs +++ b/src/installer/tests/TestUtils/Command.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.ComponentModel; using System.Diagnostics; using System.IO; using System.Linq; @@ -120,7 +121,7 @@ private static bool ShouldUseCmd(string executable) } else { - // Search the path to see if we can find it + // Search the path to see if we can find it foreach (var path in System.Environment.GetEnvironmentVariable("PATH").Split(Path.PathSeparator)) { var candidate = Path.Combine(path, executable + ".exe"); @@ -196,7 +197,20 @@ public Command Start() ReportExecBegin(); - Process.Start(); + // Retry if we hit ETXTBSY due to Linux race + // https://github.com/dotnet/runtime/issues/58964 + for (int i = 0; ; i++) + { + try + { + Process.Start(); + break; + } + catch (Win32Exception e) when (i < 4 && e.Message.Contains("Text file busy")) + { + Thread.Sleep(i * 20); + } + } if (Process.StartInfo.RedirectStandardOutput) { From dc6a86df3acd0ba8b2a358ec5296cea266b531f6 Mon Sep 17 00:00:00 2001 From: Andrew Au Date: Thu, 6 Jan 2022 10:14:17 -0800 Subject: [PATCH 11/39] Fix bug where we reference the entry #0 in the pinned plug queue (#60966) (#63351) We reference entry #0 in the pinned plug queue even if there are no pinned plugs at all and thus the pinned plug queue contains left-over data from the mark phase. The fix is to initialize saved_pinned_plug_index to a value that is invalid as a pinned plug queue index, and only use saved_pinned_plug_index as an index if is valid. Co-authored-by: Peter Sollich --- src/coreclr/gc/gc.cpp | 19 ++++++++++++++++--- src/coreclr/gc/gcpriv.h | 2 ++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index 249a723194565e..016c1849b24436 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -2725,7 +2725,7 @@ alloc_list gc_heap::poh_alloc_list [NUM_POH_ALIST-1]; #ifdef DOUBLY_LINKED_FL // size we removed with no undo; only for recording purpose size_t gc_heap::gen2_removed_no_undo = 0; -size_t gc_heap::saved_pinned_plug_index = 0; +size_t gc_heap::saved_pinned_plug_index = INVALID_SAVED_PINNED_PLUG_INDEX; #endif //DOUBLY_LINKED_FL #ifdef FEATURE_EVENT_TRACE @@ -13903,7 +13903,20 @@ void gc_heap::adjust_limit (uint8_t* start, size_t limit_size, generation* gen) uint8_t* old_loc = generation_last_free_list_allocated (gen); // check if old_loc happens to be in a saved plug_and_gap with a pinned plug after it - uint8_t* saved_plug_and_gap = pinned_plug (pinned_plug_of (saved_pinned_plug_index)) - sizeof(plug_and_gap); + uint8_t* saved_plug_and_gap = nullptr; + if (saved_pinned_plug_index != INVALID_SAVED_PINNED_PLUG_INDEX) + { + saved_plug_and_gap = pinned_plug (pinned_plug_of (saved_pinned_plug_index)) - sizeof(plug_and_gap); + + dprintf (3333, ("[h%d] sppi: %Id mtos: %Id old_loc: %Ix pp: %Ix(%Id) offs: %Id", + heap_number, + saved_pinned_plug_index, + mark_stack_tos, + old_loc, + pinned_plug (pinned_plug_of (saved_pinned_plug_index)), + pinned_len (pinned_plug_of (saved_pinned_plug_index)), + old_loc - saved_plug_and_gap)); + } size_t offset = old_loc - saved_plug_and_gap; if (offset < sizeof(gap_reloc_pair)) { @@ -27519,7 +27532,7 @@ void gc_heap::plan_phase (int condemned_gen_number) #ifdef DOUBLY_LINKED_FL gen2_removed_no_undo = 0; - saved_pinned_plug_index = 0; + saved_pinned_plug_index = INVALID_SAVED_PINNED_PLUG_INDEX; #endif //DOUBLY_LINKED_FL while (1) diff --git a/src/coreclr/gc/gcpriv.h b/src/coreclr/gc/gcpriv.h index 599e818773aca9..a04e24c112a48c 100644 --- a/src/coreclr/gc/gcpriv.h +++ b/src/coreclr/gc/gcpriv.h @@ -4496,6 +4496,8 @@ class gc_heap PER_HEAP size_t gen2_removed_no_undo; +#define INVALID_SAVED_PINNED_PLUG_INDEX ((size_t)~0) + PER_HEAP size_t saved_pinned_plug_index; #endif //DOUBLY_LINKED_FL From 4f01ff511f2d40173d6fe18e418490945cfe54f6 Mon Sep 17 00:00:00 2001 From: Koundinya Veluri Date: Thu, 6 Jan 2022 10:14:27 -0800 Subject: [PATCH 12/39] Fix named mutexes on OSX to work between arm64 and emulated x64 processes (#62765) - The page size is different between arm64 processes and emulated x64 processes - The shared memory file size is set to the page size and there was a strict check on the file size, leading to an exception from the second process of a different arch that tries to share the same mutex - Made the file size check less strict, and allowed an arch to increase but not decrease the file size such that it can be mapped at page size granularity - Fix for https://github.com/dotnet/runtime/issues/62140 in main --- src/coreclr/pal/src/include/pal/mutex.hpp | 13 +++++++-- .../pal/src/include/pal/sharedmemory.h | 3 +- .../pal/src/sharedmemory/sharedmemory.cpp | 28 ++++++++++++++----- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/src/coreclr/pal/src/include/pal/mutex.hpp b/src/coreclr/pal/src/include/pal/mutex.hpp index 8aeaf9f62586ed..8a70fd6c7cd418 100644 --- a/src/coreclr/pal/src/include/pal/mutex.hpp +++ b/src/coreclr/pal/src/include/pal/mutex.hpp @@ -120,9 +120,16 @@ Miscellaneous existing shared memory, naming, and waiting infrastructure is not suitable for this purpose, and is not used. */ -// Temporarily disabling usage of pthread process-shared mutexes on ARM/ARM64 due to functional issues that cannot easily be -// detected with code due to hangs. See https://github.com/dotnet/runtime/issues/6014. -#if HAVE_FULLY_FEATURED_PTHREAD_MUTEXES && HAVE_FUNCTIONAL_PTHREAD_ROBUST_MUTEXES && !(defined(HOST_ARM) || defined(HOST_ARM64) || defined(__FreeBSD__)) +// - Temporarily disabling usage of pthread process-shared mutexes on ARM/ARM64 due to functional issues that cannot easily be +// detected with code due to hangs. See https://github.com/dotnet/runtime/issues/6014. +// - On FreeBSD, pthread process-shared robust mutexes cannot be placed in shared memory mapped independently by the processes +// involved. See https://github.com/dotnet/runtime/issues/10519. +// - On OSX, pthread robust mutexes were/are not available at the time of this writing. In case they are made available in the +// future, their use is disabled for compatibility. +#if HAVE_FULLY_FEATURED_PTHREAD_MUTEXES && \ + HAVE_FUNCTIONAL_PTHREAD_ROBUST_MUTEXES && \ + !(defined(HOST_ARM) || defined(HOST_ARM64) || defined(__FreeBSD__) || defined(TARGET_OSX)) + #define NAMED_MUTEX_USE_PTHREAD_MUTEX 1 #else #define NAMED_MUTEX_USE_PTHREAD_MUTEX 0 diff --git a/src/coreclr/pal/src/include/pal/sharedmemory.h b/src/coreclr/pal/src/include/pal/sharedmemory.h index 1ded94e12fcc59..c6e5abe97c3a78 100644 --- a/src/coreclr/pal/src/include/pal/sharedmemory.h +++ b/src/coreclr/pal/src/include/pal/sharedmemory.h @@ -173,7 +173,8 @@ class SharedMemorySharedDataHeader }; public: - static SIZE_T DetermineTotalByteCount(SIZE_T dataByteCount); + static SIZE_T GetUsedByteCount(SIZE_T dataByteCount); + static SIZE_T GetTotalByteCount(SIZE_T dataByteCount); public: SharedMemorySharedDataHeader(SharedMemoryType type, UINT8 version); diff --git a/src/coreclr/pal/src/sharedmemory/sharedmemory.cpp b/src/coreclr/pal/src/sharedmemory/sharedmemory.cpp index 4c946cc5257b74..b1d7b3b6830459 100644 --- a/src/coreclr/pal/src/sharedmemory/sharedmemory.cpp +++ b/src/coreclr/pal/src/sharedmemory/sharedmemory.cpp @@ -519,9 +519,14 @@ bool SharedMemoryId::AppendSessionDirectoryName(PathCharString& path) const //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // SharedMemorySharedDataHeader -SIZE_T SharedMemorySharedDataHeader::DetermineTotalByteCount(SIZE_T dataByteCount) +SIZE_T SharedMemorySharedDataHeader::GetUsedByteCount(SIZE_T dataByteCount) { - return SharedMemoryHelpers::AlignUp(sizeof(SharedMemorySharedDataHeader) + dataByteCount, GetVirtualPageSize()); + return sizeof(SharedMemorySharedDataHeader) + dataByteCount; +} + +SIZE_T SharedMemorySharedDataHeader::GetTotalByteCount(SIZE_T dataByteCount) +{ + return SharedMemoryHelpers::AlignUp(GetUsedByteCount(dataByteCount), GetVirtualPageSize()); } SharedMemorySharedDataHeader::SharedMemorySharedDataHeader(SharedMemoryType type, UINT8 version) @@ -642,7 +647,7 @@ SharedMemoryProcessDataHeader *SharedMemoryProcessDataHeader::CreateOrOpen( { _ASSERTE( processDataHeader->GetSharedDataTotalByteCount() == - SharedMemorySharedDataHeader::DetermineTotalByteCount(sharedDataByteCount)); + SharedMemorySharedDataHeader::GetTotalByteCount(sharedDataByteCount)); processDataHeader->IncRefCount(); return processDataHeader; } @@ -697,14 +702,23 @@ SharedMemoryProcessDataHeader *SharedMemoryProcessDataHeader::CreateOrOpen( } // Set or validate the file length - SIZE_T sharedDataTotalByteCount = SharedMemorySharedDataHeader::DetermineTotalByteCount(sharedDataByteCount); + SIZE_T sharedDataUsedByteCount = SharedMemorySharedDataHeader::GetUsedByteCount(sharedDataByteCount); + SIZE_T sharedDataTotalByteCount = SharedMemorySharedDataHeader::GetTotalByteCount(sharedDataByteCount); if (createdFile) { SharedMemoryHelpers::SetFileSize(fileDescriptor, sharedDataTotalByteCount); } - else if (SharedMemoryHelpers::GetFileSize(fileDescriptor) != sharedDataTotalByteCount) + else { - throw SharedMemoryException(static_cast(SharedMemoryError::HeaderMismatch)); + SIZE_T currentFileSize = SharedMemoryHelpers::GetFileSize(fileDescriptor); + if (currentFileSize < sharedDataUsedByteCount) + { + throw SharedMemoryException(static_cast(SharedMemoryError::HeaderMismatch)); + } + if (currentFileSize < sharedDataTotalByteCount) + { + SharedMemoryHelpers::SetFileSize(fileDescriptor, sharedDataTotalByteCount); + } } // Acquire and hold a shared file lock on the shared memory file as long as it is open, to indicate that this process is @@ -726,7 +740,7 @@ SharedMemoryProcessDataHeader *SharedMemoryProcessDataHeader::CreateOrOpen( { if (clearContents) { - memset(mappedBuffer, 0, sharedDataTotalByteCount); + memset(mappedBuffer, 0, sharedDataUsedByteCount); } sharedDataHeader = new(mappedBuffer) SharedMemorySharedDataHeader(requiredSharedDataHeader); } From b71484cd261a6f3ae7ede71d253c2c9cd3e92b7d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 6 Jan 2022 10:46:06 -0800 Subject: [PATCH 13/39] Unify the FlushProcessWriteBuffers mechanism for macOS arm64 / x64 (#63301) The macOS arm64 was using a new mechanism to implement FlushProcessWriteBuffers because the original one based on a wired memory page doesn't work on arm64. We have seen people hitting problems with the wired memory page allocation on x64 in the past due to the fact that wired memory is a scarce resource and in some cases, e.g. in presence of apps that greedily grab all of the available wired memory pages, we were unable to initialize coreclr and execute any .NET application. This change switches x64 to the same mechanism as arm64 to prevent that issue from happening and updates the minimum supported x64 macOS to 10.14. That's the version required for the new APIs the change is using. Apple stopped supporting 10.13 a year ago anyways. Co-authored-by: Jan Vorlicek --- eng/native/configurecompiler.cmake | 2 +- src/coreclr/gc/unix/gcenv.unix.cpp | 10 ++++------ src/coreclr/pal/src/thread/process.cpp | 10 ++++------ 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/eng/native/configurecompiler.cmake b/eng/native/configurecompiler.cmake index 502d432f17bb3f..c6f4a6a5627542 100644 --- a/eng/native/configurecompiler.cmake +++ b/eng/native/configurecompiler.cmake @@ -420,7 +420,7 @@ if (CLR_CMAKE_HOST_UNIX) set(CMAKE_OSX_DEPLOYMENT_TARGET "11.0") add_compile_options(-arch arm64) elseif(CLR_CMAKE_HOST_ARCH_AMD64) - set(CMAKE_OSX_DEPLOYMENT_TARGET "10.13") + set(CMAKE_OSX_DEPLOYMENT_TARGET "10.14") add_compile_options(-arch x86_64) else() clr_unknown_arch() diff --git a/src/coreclr/gc/unix/gcenv.unix.cpp b/src/coreclr/gc/unix/gcenv.unix.cpp index 2a37c17f8c9f81..6319c519727e73 100644 --- a/src/coreclr/gc/unix/gcenv.unix.cpp +++ b/src/coreclr/gc/unix/gcenv.unix.cpp @@ -67,7 +67,6 @@ #include #include -#if defined(HOST_ARM64) #include #include extern "C" @@ -84,7 +83,6 @@ extern "C" abort(); \ } \ } while (false) -#endif // defined(HOST_ARM64) #endif // __APPLE__ @@ -372,7 +370,7 @@ bool GCToOSInterface::Initialize() { s_flushUsingMemBarrier = TRUE; } -#if !(defined(TARGET_OSX) && defined(HOST_ARM64)) +#ifndef TARGET_OSX else { assert(g_helperPage == 0); @@ -404,7 +402,7 @@ bool GCToOSInterface::Initialize() return false; } } -#endif // !(defined(TARGET_OSX) && defined(HOST_ARM64)) +#endif // !TARGET_OSX InitializeCGroup(); @@ -544,7 +542,7 @@ void GCToOSInterface::FlushProcessWriteBuffers() status = pthread_mutex_unlock(&g_flushProcessWriteBuffersMutex); assert(status == 0 && "Failed to unlock the flushProcessWriteBuffersMutex lock"); } -#if defined(TARGET_OSX) && defined(HOST_ARM64) +#ifdef TARGET_OSX else { mach_msg_type_number_t cThreads; @@ -570,7 +568,7 @@ void GCToOSInterface::FlushProcessWriteBuffers() machret = vm_deallocate(mach_task_self(), (vm_address_t)pThreads, cThreads * sizeof(thread_act_t)); CHECK_MACH("vm_deallocate()", machret); } -#endif // defined(TARGET_OSX) && defined(HOST_ARM64) +#endif // TARGET_OSX } // Break into a debugger. Uses a compiler intrinsic if one is available, diff --git a/src/coreclr/pal/src/thread/process.cpp b/src/coreclr/pal/src/thread/process.cpp index 2b349a6c37b1c2..f31449fb19c3e8 100644 --- a/src/coreclr/pal/src/thread/process.cpp +++ b/src/coreclr/pal/src/thread/process.cpp @@ -85,7 +85,6 @@ SET_DEFAULT_DEBUG_CHANNEL(PROCESS); // some headers have code with asserts, so d #include #include #include -#if defined(HOST_ARM64) #include #include extern "C" @@ -103,7 +102,6 @@ extern "C" } \ } while (false) -#endif // defined(HOST_ARM64) #endif // __APPLE__ #ifdef __NetBSD__ @@ -3472,7 +3470,7 @@ InitializeFlushProcessWriteBuffers() } } -#if defined(TARGET_OSX) && defined(HOST_ARM64) +#ifdef TARGET_OSX return TRUE; #else s_helperPage = static_cast(mmap(0, GetVirtualPageSize(), PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)); @@ -3502,7 +3500,7 @@ InitializeFlushProcessWriteBuffers() } return status == 0; -#endif // defined(TARGET_OSX) && defined(HOST_ARM64) +#endif // TARGET_OSX } #define FATAL_ASSERT(e, msg) \ @@ -3552,7 +3550,7 @@ FlushProcessWriteBuffers() status = pthread_mutex_unlock(&flushProcessWriteBuffersMutex); FATAL_ASSERT(status == 0, "Failed to unlock the flushProcessWriteBuffersMutex lock"); } -#if defined(TARGET_OSX) && defined(HOST_ARM64) +#ifdef TARGET_OSX else { mach_msg_type_number_t cThreads; @@ -3578,7 +3576,7 @@ FlushProcessWriteBuffers() machret = vm_deallocate(mach_task_self(), (vm_address_t)pThreads, cThreads * sizeof(thread_act_t)); CHECK_MACH("vm_deallocate()", machret); } -#endif // defined(TARGET_OSX) && defined(HOST_ARM64) +#endif // TARGET_OSX } /*++ From 898f09abb3737c47d7bde16678b1ed8359982be5 Mon Sep 17 00:00:00 2001 From: Jan Vorlicek Date: Thu, 6 Jan 2022 20:07:37 +0100 Subject: [PATCH 14/39] [release/6.0] Change macOS activatin injection failure handling (#63393) Backport of #59045 to release/6.0 The pthread_kill can fail with ENOTSUP on macOS when the target thread is a dispatch queue thread. Instead of aborting the process, it is better to fail to inject the activation and rely on return address hijacking and other means of syncing with GC. --- src/coreclr/pal/src/exception/signal.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/coreclr/pal/src/exception/signal.cpp b/src/coreclr/pal/src/exception/signal.cpp index cbcd1cac6f226e..59b513b87204e9 100644 --- a/src/coreclr/pal/src/exception/signal.cpp +++ b/src/coreclr/pal/src/exception/signal.cpp @@ -845,6 +845,15 @@ PAL_ERROR InjectActivationInternal(CorUnix::CPalThread* pThread) // We can get EAGAIN when printing stack overflow stack trace and when other threads hit // stack overflow too. Those are held in the sigsegv_handler with blocked signals until // the process exits. + +#ifdef __APPLE__ + // On Apple, pthread_kill is not allowed to be sent to dispatch queue threads + if (status == ENOTSUP) + { + return ERROR_NOT_SUPPORTED; + } +#endif + if ((status != 0) && (status != EAGAIN)) { // Failure to send the signal is fatal. There are only two cases when sending From 1c2501d8dfa64af395e0f6428063df9b8563757a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 6 Jan 2022 12:04:56 -0800 Subject: [PATCH 15/39] [release/6.0] Fix XXHash for stripe size (#61923) * Fix XXHash for stripe size * Add servicing properties to System.IO.Hashing * Run package validation against 6.0.0 packages and enable it for IO Hashing. Co-authored-by: Tornhoof Co-authored-by: Jeremy Barton Co-authored-by: Santiago Fernandez Madero --- eng/packaging.targets | 2 +- .../System.IO.Hashing/src/System.IO.Hashing.csproj | 4 ++-- .../System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs | 2 +- .../System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs | 2 +- src/libraries/System.IO.Hashing/tests/XxHash32Tests.007.cs | 7 +++++++ src/libraries/System.IO.Hashing/tests/XxHash32Tests.cs | 7 +++++++ .../System.IO.Hashing/tests/XxHash32Tests.f00d.cs | 7 +++++++ src/libraries/System.IO.Hashing/tests/XxHash64Tests.007.cs | 7 +++++++ src/libraries/System.IO.Hashing/tests/XxHash64Tests.cs | 7 +++++++ .../System.IO.Hashing/tests/XxHash64Tests.f00d.cs | 7 +++++++ 10 files changed, 47 insertions(+), 5 deletions(-) diff --git a/eng/packaging.targets b/eng/packaging.targets index 9fe4b569e5f703..6bcf86dc9f2e39 100644 --- a/eng/packaging.targets +++ b/eng/packaging.targets @@ -10,7 +10,7 @@ true true - $([MSBuild]::Subtract($(MajorVersion), 1)).0.0 + 6.0.0 AddNETStandardCompatErrorFileForPackaging;IncludeAnalyzersInPackage;$(PackDependsOn) diff --git a/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj b/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj index acfa86bb798d87..079a554ac9db27 100644 --- a/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj +++ b/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj @@ -4,8 +4,8 @@ enable $(NetCoreAppCurrent);netstandard2.0;net461 true - - true + true + 1 Provides non-cryptographic hash algorithms, such as CRC-32. Commonly Used Types: diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs index d6db9bae19b2e3..792a5bbbffb8df 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs @@ -215,7 +215,7 @@ private static int StaticHash(ReadOnlySpan source, Span destination, int totalLength = source.Length; State state = new State((uint)seed); - while (source.Length > StripeSize) + while (source.Length >= StripeSize) { state.ProcessStripe(source); source = source.Slice(StripeSize); diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs index 990ed77d64de27..367c6213c25fe3 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs @@ -215,7 +215,7 @@ private static int StaticHash(ReadOnlySpan source, Span destination, int totalLength = source.Length; State state = new State((ulong)seed); - while (source.Length > StripeSize) + while (source.Length >= StripeSize) { state.ProcessStripe(source); source = source.Slice(StripeSize); diff --git a/src/libraries/System.IO.Hashing/tests/XxHash32Tests.007.cs b/src/libraries/System.IO.Hashing/tests/XxHash32Tests.007.cs index eed5aec365b3eb..9ec9262dd1c5d8 100644 --- a/src/libraries/System.IO.Hashing/tests/XxHash32Tests.007.cs +++ b/src/libraries/System.IO.Hashing/tests/XxHash32Tests.007.cs @@ -36,6 +36,8 @@ public static IEnumerable TestCases private const string DotNetHashesThis3 = DotNetHashesThis + DotNetHashesThis + DotNetHashesThis; private const string DotNetNCHashing = ".NET now has non-crypto hashing"; private const string DotNetNCHashing3 = DotNetNCHashing + DotNetNCHashing + DotNetNCHashing; + private const string SixteenBytes = ".NET Hashes This"; + private const string SixteenBytes3 = SixteenBytes + SixteenBytes + SixteenBytes; protected static IEnumerable TestCaseDefinitions { get; } = new[] @@ -77,6 +79,11 @@ public static IEnumerable TestCases $"{DotNetNCHashing} (x3)", Encoding.ASCII.GetBytes(DotNetNCHashing3), "CABC8ABD"), + // stripe size + new TestCase( + $"{SixteenBytes} (x3)", + Encoding.ASCII.GetBytes(SixteenBytes3), + "AD98EBD3") }; protected override NonCryptographicHashAlgorithm CreateInstance() => new XxHash32(Seed); diff --git a/src/libraries/System.IO.Hashing/tests/XxHash32Tests.cs b/src/libraries/System.IO.Hashing/tests/XxHash32Tests.cs index f9dccad0294028..abd77d8c22f97d 100644 --- a/src/libraries/System.IO.Hashing/tests/XxHash32Tests.cs +++ b/src/libraries/System.IO.Hashing/tests/XxHash32Tests.cs @@ -34,6 +34,8 @@ public static IEnumerable TestCases private const string DotNetHashesThis3 = DotNetHashesThis + DotNetHashesThis + DotNetHashesThis; private const string DotNetNCHashing = ".NET now has non-crypto hashing"; private const string DotNetNCHashing3 = DotNetNCHashing + DotNetNCHashing + DotNetNCHashing; + private const string SixteenBytes = ".NET Hashes This"; + private const string SixteenBytes3 = SixteenBytes + SixteenBytes + SixteenBytes; protected static IEnumerable TestCaseDefinitions { get; } = new[] @@ -90,6 +92,11 @@ public static IEnumerable TestCases $"{DotNetNCHashing} (x3)", Encoding.ASCII.GetBytes(DotNetNCHashing3), "65242024"), + // stripe size + new TestCase( + $"{SixteenBytes} (x3)", + Encoding.ASCII.GetBytes(SixteenBytes3), + "29DA7472") }; protected override NonCryptographicHashAlgorithm CreateInstance() => new XxHash32(); diff --git a/src/libraries/System.IO.Hashing/tests/XxHash32Tests.f00d.cs b/src/libraries/System.IO.Hashing/tests/XxHash32Tests.f00d.cs index 5171e62f8ca562..4a116bc50354e0 100644 --- a/src/libraries/System.IO.Hashing/tests/XxHash32Tests.f00d.cs +++ b/src/libraries/System.IO.Hashing/tests/XxHash32Tests.f00d.cs @@ -36,6 +36,8 @@ public static IEnumerable TestCases private const string DotNetHashesThis3 = DotNetHashesThis + DotNetHashesThis + DotNetHashesThis; private const string DotNetNCHashing = ".NET now has non-crypto hashing"; private const string DotNetNCHashing3 = DotNetNCHashing + DotNetNCHashing + DotNetNCHashing; + private const string SixteenBytes = ".NET Hashes This"; + private const string SixteenBytes3 = SixteenBytes + SixteenBytes + SixteenBytes; protected static IEnumerable TestCaseDefinitions { get; } = new[] @@ -77,6 +79,11 @@ public static IEnumerable TestCases $"{DotNetNCHashing} (x3)", Encoding.ASCII.GetBytes(DotNetNCHashing3), "5A513E6D"), + // stripe size + new TestCase( + $"{SixteenBytes} (x3)", + Encoding.ASCII.GetBytes(SixteenBytes3), + "B38A9A45") }; protected override NonCryptographicHashAlgorithm CreateInstance() => new XxHash32(Seed); diff --git a/src/libraries/System.IO.Hashing/tests/XxHash64Tests.007.cs b/src/libraries/System.IO.Hashing/tests/XxHash64Tests.007.cs index 5c2e575377870d..836b29230f3e67 100644 --- a/src/libraries/System.IO.Hashing/tests/XxHash64Tests.007.cs +++ b/src/libraries/System.IO.Hashing/tests/XxHash64Tests.007.cs @@ -38,6 +38,8 @@ public static IEnumerable TestCases private const string DotNetNCHashing = ".NET now has non-crypto hashing"; private const string SixtyThreeBytes = "A sixty-three byte test input requires substantial forethought!"; private const string SixtyThreeBytes3 = SixtyThreeBytes + SixtyThreeBytes + SixtyThreeBytes; + private const string ThirtyTwoBytes = "This string has 32 ASCII bytes.."; + private const string ThirtyTwoBytes3 = ThirtyTwoBytes + ThirtyTwoBytes + ThirtyTwoBytes; protected static IEnumerable TestCaseDefinitions { get; } = new[] @@ -87,6 +89,11 @@ public static IEnumerable TestCases $"{SixtyThreeBytes} (x3)", Encoding.ASCII.GetBytes(SixtyThreeBytes3), "D6095B93EB10BEDA"), + // stripe size + new TestCase( + $"{ThirtyTwoBytes} (x3)", + Encoding.ASCII.GetBytes(ThirtyTwoBytes3), + "45116421CF932B1F") }; protected override NonCryptographicHashAlgorithm CreateInstance() => new XxHash64(Seed); diff --git a/src/libraries/System.IO.Hashing/tests/XxHash64Tests.cs b/src/libraries/System.IO.Hashing/tests/XxHash64Tests.cs index 973d108fc93778..213e6f91c252cf 100644 --- a/src/libraries/System.IO.Hashing/tests/XxHash64Tests.cs +++ b/src/libraries/System.IO.Hashing/tests/XxHash64Tests.cs @@ -36,6 +36,8 @@ public static IEnumerable TestCases private const string DotNetNCHashing = ".NET now has non-crypto hashing"; private const string SixtyThreeBytes = "A sixty-three byte test input requires substantial forethought!"; private const string SixtyThreeBytes3 = SixtyThreeBytes + SixtyThreeBytes + SixtyThreeBytes; + private const string ThirtyTwoBytes = "This string has 32 ASCII bytes.."; + private const string ThirtyTwoBytes3 = ThirtyTwoBytes + ThirtyTwoBytes + ThirtyTwoBytes; protected static IEnumerable TestCaseDefinitions { get; } = new[] @@ -103,6 +105,11 @@ public static IEnumerable TestCases $"{SixtyThreeBytes} (x3)", Encoding.ASCII.GetBytes(SixtyThreeBytes3), "239C7B3A85BD22B3"), + // stripe size + new TestCase( + $"{ThirtyTwoBytes} (x3)", + Encoding.ASCII.GetBytes(ThirtyTwoBytes3), + "975E3E6FE7E67FBC") }; protected override NonCryptographicHashAlgorithm CreateInstance() => new XxHash64(); diff --git a/src/libraries/System.IO.Hashing/tests/XxHash64Tests.f00d.cs b/src/libraries/System.IO.Hashing/tests/XxHash64Tests.f00d.cs index 23006a571627b3..75fc4f514aca9a 100644 --- a/src/libraries/System.IO.Hashing/tests/XxHash64Tests.f00d.cs +++ b/src/libraries/System.IO.Hashing/tests/XxHash64Tests.f00d.cs @@ -38,6 +38,8 @@ public static IEnumerable TestCases private const string DotNetNCHashing = ".NET now has non-crypto hashing"; private const string SixtyThreeBytes = "A sixty-three byte test input requires substantial forethought!"; private const string SixtyThreeBytes3 = SixtyThreeBytes + SixtyThreeBytes + SixtyThreeBytes; + private const string ThirtyTwoBytes = "This string has 32 ASCII bytes.."; + private const string ThirtyTwoBytes3 = ThirtyTwoBytes + ThirtyTwoBytes + ThirtyTwoBytes; protected static IEnumerable TestCaseDefinitions { get; } = new[] @@ -87,6 +89,11 @@ public static IEnumerable TestCases $"{SixtyThreeBytes} (x3)", Encoding.ASCII.GetBytes(SixtyThreeBytes3), "6F1C62EB48EA2FEC"), + // stripe size + new TestCase( + $"{ThirtyTwoBytes} (x3)", + Encoding.ASCII.GetBytes(ThirtyTwoBytes3), + "B358EB96B8E3E7AD") }; protected override NonCryptographicHashAlgorithm CreateInstance() => new XxHash64(Seed); From 25d2a1fa84a78aea644ad878481cb307855e60c2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 6 Jan 2022 16:55:37 -0700 Subject: [PATCH 16/39] [release/6.0] Making user secrets optional by default (#62917) * Making user secrets optional by default * update test * Update fix for 6.0.x servicing. Co-authored-by: Maryam Ariyan Co-authored-by: Eric Erhardt --- ...crosoft.Extensions.Configuration.UserSecrets.csproj | 2 ++ .../src/UserSecretsConfigurationExtensions.cs | 4 ++-- .../tests/ConfigurationExtensionTest.cs | 10 +++++----- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/libraries/Microsoft.Extensions.Configuration.UserSecrets/src/Microsoft.Extensions.Configuration.UserSecrets.csproj b/src/libraries/Microsoft.Extensions.Configuration.UserSecrets/src/Microsoft.Extensions.Configuration.UserSecrets.csproj index d488eecf483e74..3fda47281fe0f4 100644 --- a/src/libraries/Microsoft.Extensions.Configuration.UserSecrets/src/Microsoft.Extensions.Configuration.UserSecrets.csproj +++ b/src/libraries/Microsoft.Extensions.Configuration.UserSecrets/src/Microsoft.Extensions.Configuration.UserSecrets.csproj @@ -4,6 +4,8 @@ netstandard2.0;net461 true User secrets configuration provider implementation for Microsoft.Extensions.Configuration. + true + 1 diff --git a/src/libraries/Microsoft.Extensions.Configuration.UserSecrets/src/UserSecretsConfigurationExtensions.cs b/src/libraries/Microsoft.Extensions.Configuration.UserSecrets/src/UserSecretsConfigurationExtensions.cs index 86139faf9b021d..963ac77a30eea6 100644 --- a/src/libraries/Microsoft.Extensions.Configuration.UserSecrets/src/UserSecretsConfigurationExtensions.cs +++ b/src/libraries/Microsoft.Extensions.Configuration.UserSecrets/src/UserSecretsConfigurationExtensions.cs @@ -29,7 +29,7 @@ public static class UserSecretsConfigurationExtensions /// The configuration builder. public static IConfigurationBuilder AddUserSecrets(this IConfigurationBuilder configuration) where T : class - => configuration.AddUserSecrets(typeof(T).Assembly, optional: false, reloadOnChange: false); + => configuration.AddUserSecrets(typeof(T).Assembly, optional: true, reloadOnChange: false); /// /// @@ -82,7 +82,7 @@ public static IConfigurationBuilder AddUserSecrets(this IConfigurationBuilder /// Thrown when does not have a valid /// The configuration builder. public static IConfigurationBuilder AddUserSecrets(this IConfigurationBuilder configuration, Assembly assembly) - => configuration.AddUserSecrets(assembly, optional: false, reloadOnChange: false); + => configuration.AddUserSecrets(assembly, optional: true, reloadOnChange: false); /// /// diff --git a/src/libraries/Microsoft.Extensions.Configuration.UserSecrets/tests/ConfigurationExtensionTest.cs b/src/libraries/Microsoft.Extensions.Configuration.UserSecrets/tests/ConfigurationExtensionTest.cs index 03667ab17b33c5..53dde10c61df4a 100644 --- a/src/libraries/Microsoft.Extensions.Configuration.UserSecrets/tests/ConfigurationExtensionTest.cs +++ b/src/libraries/Microsoft.Extensions.Configuration.UserSecrets/tests/ConfigurationExtensionTest.cs @@ -85,23 +85,23 @@ public void AddUserSecrets_FindsAssemblyAttributeFromType() public void AddUserSecrets_ThrowsIfAssemblyAttributeFromType() { var ex = Assert.Throws(() => - new ConfigurationBuilder().AddUserSecrets()); + new ConfigurationBuilder().AddUserSecrets(optional: false)); Assert.Equal(SR.Format(SR.Error_Missing_UserSecretsIdAttribute, typeof(string).Assembly.GetName().Name), ex.Message); ex = Assert.Throws(() => - new ConfigurationBuilder().AddUserSecrets(typeof(JObject).Assembly)); + new ConfigurationBuilder().AddUserSecrets(typeof(JObject).Assembly, optional: false)); Assert.Equal(SR.Format(SR.Error_Missing_UserSecretsIdAttribute, typeof(JObject).Assembly.GetName().Name), ex.Message); } [Fact] - public void AddUserSecrets_DoesNotThrowsIfOptional() + public void AddUserSecrets_DoesNotThrowsIfOptionalByDefault() { var config = new ConfigurationBuilder() - .AddUserSecrets(optional: true) - .AddUserSecrets(typeof(List<>).Assembly, optional: true) + .AddUserSecrets() + .AddUserSecrets(typeof(List<>).Assembly) .Build(); Assert.Empty(config.AsEnumerable()); From 201673a9c7a24d1a375533e185610bbef2a0459d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 7 Jan 2022 09:37:14 -0700 Subject: [PATCH 17/39] [release/6.0] WinHttp: always read HTTP/2 streams to the end (#63346) * tests * enable WINHTTP_OPTION_REQUIRE_STREAM_END on request handles * set WINHTTP_OPTION_REQUIRE_STREAM_END on the session handle Co-authored-by: antonfirsov --- .../Windows/WinHttp/Interop.winhttp_types.cs | 1 + .../src/System/Net/Http/WinHttpHandler.cs | 22 ++++++++++++++ .../FunctionalTests/TrailingHeadersTest.cs | 30 +++++++++++++++---- .../FunctionalTests/SocketsHttpHandlerTest.cs | 30 +++++++++++++++---- 4 files changed, 71 insertions(+), 12 deletions(-) diff --git a/src/libraries/Common/src/Interop/Windows/WinHttp/Interop.winhttp_types.cs b/src/libraries/Common/src/Interop/Windows/WinHttp/Interop.winhttp_types.cs index 3344e34b0265e3..463de2572b2b3f 100644 --- a/src/libraries/Common/src/Interop/Windows/WinHttp/Interop.winhttp_types.cs +++ b/src/libraries/Common/src/Interop/Windows/WinHttp/Interop.winhttp_types.cs @@ -168,6 +168,7 @@ internal static partial class WinHttp public const uint WINHTTP_OPTION_TCP_KEEPALIVE = 152; public const uint WINHTTP_OPTION_STREAM_ERROR_CODE = 159; + public const uint WINHTTP_OPTION_REQUIRE_STREAM_END = 160; public enum WINHTTP_WEB_SOCKET_BUFFER_TYPE { diff --git a/src/libraries/System.Net.Http.WinHttpHandler/src/System/Net/Http/WinHttpHandler.cs b/src/libraries/System.Net.Http.WinHttpHandler/src/System/Net/Http/WinHttpHandler.cs index 3b4c247b8b7032..07133b2667c229 100644 --- a/src/libraries/System.Net.Http.WinHttpHandler/src/System/Net/Http/WinHttpHandler.cs +++ b/src/libraries/System.Net.Http.WinHttpHandler/src/System/Net/Http/WinHttpHandler.cs @@ -1120,6 +1120,7 @@ private void SetSessionHandleOptions(SafeWinHttpHandle sessionHandle) SetSessionHandleTimeoutOptions(sessionHandle); SetDisableHttp2StreamQueue(sessionHandle); SetTcpKeepalive(sessionHandle); + SetRequireStreamEnd(sessionHandle); } private unsafe void SetTcpKeepalive(SafeWinHttpHandle sessionHandle) @@ -1145,6 +1146,27 @@ private unsafe void SetTcpKeepalive(SafeWinHttpHandle sessionHandle) } } + private void SetRequireStreamEnd(SafeWinHttpHandle sessionHandle) + { + if (WinHttpTrailersHelper.OsSupportsTrailers) + { + // Setting WINHTTP_OPTION_REQUIRE_STREAM_END to TRUE is needed for WinHttp to read trailing headers + // in case the response has Content-Lenght defined. + // According to the WinHttp team, the feature-detection logic in WinHttpTrailersHelper.OsSupportsTrailers + // should also indicate the support of WINHTTP_OPTION_REQUIRE_STREAM_END. + // WINHTTP_OPTION_REQUIRE_STREAM_END doesn't have effect on HTTP 1.1 requests, therefore it's safe to set it on + // the session handle so it is inhereted by all request handles. + uint optionData = 1; + if (!Interop.WinHttp.WinHttpSetOption(sessionHandle, Interop.WinHttp.WINHTTP_OPTION_REQUIRE_STREAM_END, ref optionData)) + { + if (NetEventSource.Log.IsEnabled()) + { + NetEventSource.Info(this, "Failed to enable WINHTTP_OPTION_REQUIRE_STREAM_END error code: " + Marshal.GetLastWin32Error()); + } + } + } + } + private void SetSessionHandleConnectionOptions(SafeWinHttpHandle sessionHandle) { uint optionData = (uint)_maxConnectionsPerServer; diff --git a/src/libraries/System.Net.Http.WinHttpHandler/tests/FunctionalTests/TrailingHeadersTest.cs b/src/libraries/System.Net.Http.WinHttpHandler/tests/FunctionalTests/TrailingHeadersTest.cs index cef52465524dcf..5ac753c266ee0a 100644 --- a/src/libraries/System.Net.Http.WinHttpHandler/tests/FunctionalTests/TrailingHeadersTest.cs +++ b/src/libraries/System.Net.Http.WinHttpHandler/tests/FunctionalTests/TrailingHeadersTest.cs @@ -67,8 +67,10 @@ public async Task Http2GetAsync_NoTrailingHeaders_EmptyCollection() } } - [ConditionalFact(nameof(TestsEnabled))] - public async Task Http2GetAsync_MissingTrailer_TrailingHeadersAccepted() + [InlineData(false)] + [InlineData(true)] + [ConditionalTheory(nameof(TestsEnabled))] + public async Task Http2GetAsync_MissingTrailer_TrailingHeadersAccepted(bool responseHasContentLength) { using (Http2LoopbackServer server = Http2LoopbackServer.CreateServer()) using (HttpClient client = CreateHttpClient()) @@ -80,7 +82,14 @@ public async Task Http2GetAsync_MissingTrailer_TrailingHeadersAccepted() int streamId = await connection.ReadRequestHeaderAsync(); // Response header. - await connection.SendDefaultResponseHeadersAsync(streamId); + if (responseHasContentLength) + { + await connection.SendResponseHeadersAsync(streamId, endStream: false, headers: new[] { new HttpHeaderData("Content-Length", DataBytes.Length.ToString()) }); + } + else + { + await connection.SendDefaultResponseHeadersAsync(streamId); + } // Response data, missing Trailers. await connection.WriteFrameAsync(MakeDataFrame(streamId, DataBytes)); @@ -98,8 +107,10 @@ public async Task Http2GetAsync_MissingTrailer_TrailingHeadersAccepted() } } - [ConditionalFact(nameof(TestsEnabled))] - public async Task Http2GetAsyncResponseHeadersReadOption_TrailingHeaders_Available() + [InlineData(false)] + [InlineData(true)] + [ConditionalTheory(nameof(TestsEnabled))] + public async Task Http2GetAsyncResponseHeadersReadOption_TrailingHeaders_Available(bool responseHasContentLength) { using (Http2LoopbackServer server = Http2LoopbackServer.CreateServer()) using (HttpClient client = CreateHttpClient()) @@ -111,7 +122,14 @@ public async Task Http2GetAsyncResponseHeadersReadOption_TrailingHeaders_Availab int streamId = await connection.ReadRequestHeaderAsync(); // Response header. - await connection.SendDefaultResponseHeadersAsync(streamId); + if (responseHasContentLength) + { + await connection.SendResponseHeadersAsync(streamId, endStream: false, headers: new[] { new HttpHeaderData("Content-Length", DataBytes.Length.ToString()) }); + } + else + { + await connection.SendDefaultResponseHeadersAsync(streamId); + } // Response data, missing Trailers. await connection.WriteFrameAsync(MakeDataFrame(streamId, DataBytes)); diff --git a/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.cs b/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.cs index 14f444b1935a60..bfd1e83bfad502 100644 --- a/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.cs +++ b/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.cs @@ -837,8 +837,10 @@ public async Task Http2GetAsync_NoTrailingHeaders_EmptyCollection() } } - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.SupportsAlpn))] - public async Task Http2GetAsync_MissingTrailer_TrailingHeadersAccepted() + [InlineData(false)] + [InlineData(true)] + [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.SupportsAlpn))] + public async Task Http2GetAsync_MissingTrailer_TrailingHeadersAccepted(bool responseHasContentLength) { using (Http2LoopbackServer server = Http2LoopbackServer.CreateServer()) using (HttpClient client = CreateHttpClient()) @@ -850,7 +852,14 @@ public async Task Http2GetAsync_MissingTrailer_TrailingHeadersAccepted() int streamId = await connection.ReadRequestHeaderAsync(); // Response header. - await connection.SendDefaultResponseHeadersAsync(streamId); + if (responseHasContentLength) + { + await connection.SendResponseHeadersAsync(streamId, endStream: false, headers: new[] { new HttpHeaderData("Content-Length", DataBytes.Length.ToString()) }); + } + else + { + await connection.SendDefaultResponseHeadersAsync(streamId); + } // Response data, missing Trailers. await connection.WriteFrameAsync(MakeDataFrame(streamId, DataBytes)); @@ -888,8 +897,10 @@ public async Task Http2GetAsync_TrailerHeaders_TrailingPseudoHeadersThrow() } } - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.SupportsAlpn))] - public async Task Http2GetAsyncResponseHeadersReadOption_TrailingHeaders_Available() + [InlineData(false)] + [InlineData(true)] + [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.SupportsAlpn))] + public async Task Http2GetAsyncResponseHeadersReadOption_TrailingHeaders_Available(bool responseHasContentLength) { using (Http2LoopbackServer server = Http2LoopbackServer.CreateServer()) using (HttpClient client = CreateHttpClient()) @@ -901,7 +912,14 @@ public async Task Http2GetAsyncResponseHeadersReadOption_TrailingHeaders_Availab int streamId = await connection.ReadRequestHeaderAsync(); // Response header. - await connection.SendDefaultResponseHeadersAsync(streamId); + if (responseHasContentLength) + { + await connection.SendResponseHeadersAsync(streamId, endStream: false, headers: new[] { new HttpHeaderData("Content-Length", DataBytes.Length.ToString()) }); + } + else + { + await connection.SendDefaultResponseHeadersAsync(streamId); + } // Response data, missing Trailers. await connection.WriteFrameAsync(MakeDataFrame(streamId, DataBytes)); From 73f902f4ab2df2404283d3711a1f765e086821e3 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 7 Jan 2022 17:38:04 +0100 Subject: [PATCH 18/39] [release/6.0] Fix setting timestamp on Windows on readonly files (#62922) * Fix setting timestamp on Windows on readonly files (#62638) # Conflicts: # src/libraries/System.IO.FileSystem/tests/Base/BaseGetSetTimes.cs * make it compile * fix build Co-authored-by: Dan Moseley --- .../Kernel32/Interop.FileOperations.cs | 3 +- .../tests/Base/BaseGetSetTimes.cs | 28 +++++++++++++++---- .../tests/Directory/GetSetTimes.cs | 4 ++- .../tests/DirectoryInfo/GetSetTimes.cs | 4 ++- .../tests/File/GetSetTimes.cs | 10 ++++++- .../tests/FileInfo/GetSetTimes.cs | 10 ++++++- .../src/System/IO/FileSystem.Windows.cs | 9 +++--- 7 files changed, 53 insertions(+), 15 deletions(-) diff --git a/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.FileOperations.cs b/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.FileOperations.cs index cc4896c1c52e48..c43b4fd19d21d2 100644 --- a/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.FileOperations.cs +++ b/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.FileOperations.cs @@ -23,7 +23,8 @@ internal static partial class FileOperations internal const int FILE_FLAG_OVERLAPPED = 0x40000000; internal const int FILE_LIST_DIRECTORY = 0x0001; - } + internal const int FILE_WRITE_ATTRIBUTES = 0x100; + } } } diff --git a/src/libraries/System.IO.FileSystem/tests/Base/BaseGetSetTimes.cs b/src/libraries/System.IO.FileSystem/tests/Base/BaseGetSetTimes.cs index a959e317682f4e..d55171847a0b7d 100644 --- a/src/libraries/System.IO.FileSystem/tests/Base/BaseGetSetTimes.cs +++ b/src/libraries/System.IO.FileSystem/tests/Base/BaseGetSetTimes.cs @@ -20,7 +20,9 @@ public abstract class BaseGetSetTimes : FileSystemTest protected static bool LowTemporalResolution => PlatformDetection.IsBrowser || isHFS; protected static bool HighTemporalResolution => !LowTemporalResolution; - protected abstract T GetExistingItem(); + protected abstract bool CanBeReadOnly { get; } + + protected abstract T GetExistingItem(bool readOnly = false); protected abstract T GetMissingItem(); protected abstract string GetItemPath(T item); @@ -42,11 +44,8 @@ public static TimeFunction Create(SetTime setter, GetTime getter, DateTimeKind k public DateTimeKind Kind => Item3; } - [Fact] - public void SettingUpdatesProperties() + private void SettingUpdatesPropertiesCore(T item) { - T item = GetExistingItem(); - Assert.All(TimeFunctions(requiresRoundtripping: true), (function) => { // Checking that milliseconds are not dropped after setter. @@ -70,6 +69,25 @@ public void SettingUpdatesProperties() }); } + [Fact] + public void SettingUpdatesProperties() + { + T item = GetExistingItem(); + SettingUpdatesPropertiesCore(item); + } + + [Fact] + public void SettingUpdatesPropertiesWhenReadOnly() + { + if (!CanBeReadOnly) + { + return; // directories can't be read only, so automatic pass + } + + T item = GetExistingItem(readOnly: true); + SettingUpdatesPropertiesCore(item); + } + [Fact] public void CanGetAllTimesAfterCreation() { diff --git a/src/libraries/System.IO.FileSystem/tests/Directory/GetSetTimes.cs b/src/libraries/System.IO.FileSystem/tests/Directory/GetSetTimes.cs index 0a23d638c138af..1fc00ba74249d6 100644 --- a/src/libraries/System.IO.FileSystem/tests/Directory/GetSetTimes.cs +++ b/src/libraries/System.IO.FileSystem/tests/Directory/GetSetTimes.cs @@ -7,7 +7,9 @@ namespace System.IO.Tests { public class Directory_GetSetTimes : StaticGetSetTimes { - protected override string GetExistingItem() => Directory.CreateDirectory(GetTestFilePath()).FullName; + protected override bool CanBeReadOnly => false; + + protected override string GetExistingItem(bool _) => Directory.CreateDirectory(GetTestFilePath()).FullName; public override IEnumerable TimeFunctions(bool requiresRoundtripping = false) { diff --git a/src/libraries/System.IO.FileSystem/tests/DirectoryInfo/GetSetTimes.cs b/src/libraries/System.IO.FileSystem/tests/DirectoryInfo/GetSetTimes.cs index 2dad268a79a460..c45c6169ada958 100644 --- a/src/libraries/System.IO.FileSystem/tests/DirectoryInfo/GetSetTimes.cs +++ b/src/libraries/System.IO.FileSystem/tests/DirectoryInfo/GetSetTimes.cs @@ -7,7 +7,9 @@ namespace System.IO.Tests { public class DirectoryInfo_GetSetTimes : InfoGetSetTimes { - protected override DirectoryInfo GetExistingItem() => Directory.CreateDirectory(GetTestFilePath()); + protected override bool CanBeReadOnly => false; + + protected override DirectoryInfo GetExistingItem(bool _) => Directory.CreateDirectory(GetTestFilePath()); protected override DirectoryInfo GetMissingItem() => new DirectoryInfo(GetTestFilePath()); diff --git a/src/libraries/System.IO.FileSystem/tests/File/GetSetTimes.cs b/src/libraries/System.IO.FileSystem/tests/File/GetSetTimes.cs index 67a13bcf7df2b5..f27e79da3f565a 100644 --- a/src/libraries/System.IO.FileSystem/tests/File/GetSetTimes.cs +++ b/src/libraries/System.IO.FileSystem/tests/File/GetSetTimes.cs @@ -11,14 +11,22 @@ namespace System.IO.Tests { public class File_GetSetTimes : StaticGetSetTimes { + protected override bool CanBeReadOnly => true; + // OSX has the limitation of setting upto 2262-04-11T23:47:16 (long.Max) date. // 32bit Unix has time_t up to ~ 2038. private static bool SupportsLongMaxDateTime => PlatformDetection.IsWindows || (!PlatformDetection.Is32BitProcess && !PlatformDetection.IsOSXLike); - protected override string GetExistingItem() + protected override string GetExistingItem(bool readOnly = false) { string path = GetTestFilePath(); File.Create(path).Dispose(); + + if (readOnly) + { + File.SetAttributes(path, FileAttributes.ReadOnly); + } + return path; } diff --git a/src/libraries/System.IO.FileSystem/tests/FileInfo/GetSetTimes.cs b/src/libraries/System.IO.FileSystem/tests/FileInfo/GetSetTimes.cs index d3b9764951cbfd..1ca30617292ebd 100644 --- a/src/libraries/System.IO.FileSystem/tests/FileInfo/GetSetTimes.cs +++ b/src/libraries/System.IO.FileSystem/tests/FileInfo/GetSetTimes.cs @@ -10,10 +10,18 @@ namespace System.IO.Tests { public class FileInfo_GetSetTimes : InfoGetSetTimes { - protected override FileInfo GetExistingItem() + protected override bool CanBeReadOnly => true; + + protected override FileInfo GetExistingItem(bool readOnly = false) { string path = GetTestFilePath(); File.Create(path).Dispose(); + + if (readOnly) + { + File.SetAttributes(path, FileAttributes.ReadOnly); + } + return new FileInfo(path); } diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/FileSystem.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/FileSystem.Windows.cs index a233347d949edc..b1ba36322ad367 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/FileSystem.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/FileSystem.Windows.cs @@ -145,10 +145,9 @@ public static void MoveFile(string sourceFullPath, string destFullPath, bool ove } } - private static SafeFileHandle OpenHandle(string fullPath, bool asDirectory) + private static SafeFileHandle OpenHandleToWriteAttributes(string fullPath, bool asDirectory) { - string root = fullPath.Substring(0, PathInternal.GetRootLength(fullPath.AsSpan())); - if (root == fullPath && root[1] == Path.VolumeSeparatorChar) + if (fullPath.Length == PathInternal.GetRootLength(fullPath.AsSpan()) && fullPath[1] == Path.VolumeSeparatorChar) { // intentionally not fullpath, most upstack public APIs expose this as path. throw new ArgumentException(SR.Arg_PathIsVolume, "path"); @@ -156,7 +155,7 @@ private static SafeFileHandle OpenHandle(string fullPath, bool asDirectory) SafeFileHandle handle = Interop.Kernel32.CreateFile( fullPath, - Interop.Kernel32.GenericOperations.GENERIC_WRITE, + Interop.Kernel32.FileOperations.FILE_WRITE_ATTRIBUTES, FileShare.ReadWrite | FileShare.Delete, FileMode.Open, asDirectory ? Interop.Kernel32.FileOperations.FILE_FLAG_BACKUP_SEMANTICS : 0); @@ -382,7 +381,7 @@ private static unsafe void SetFileTime( long changeTime = -1, uint fileAttributes = 0) { - using (SafeFileHandle handle = OpenHandle(fullPath, asDirectory)) + using (SafeFileHandle handle = OpenHandleToWriteAttributes(fullPath, asDirectory)) { var basicInfo = new Interop.Kernel32.FILE_BASIC_INFO() { From 8abc8fc289b3b4298a4672579a65e45d433a5067 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 7 Jan 2022 09:39:43 -0700 Subject: [PATCH 19/39] [release/6.0] Undo breaking change in 6.0 in environment variable configuration prefix support (#62916) * Undo breaking change in 6.0 With environment variable configuration provider, undo support for normalizing prefix using double underscore * Update src/libraries/Microsoft.Extensions.Configuration.EnvironmentVariables/tests/EnvironmentVariablesTest.cs * Update src/libraries/Microsoft.Extensions.Configuration.EnvironmentVariables/tests/EnvironmentVariablesTest.cs * update tests, not windows specific * Update fix for 6.0.x servicing. Co-authored-by: Maryam Ariyan Co-authored-by: Eric Erhardt --- ...vironmentVariablesConfigurationProvider.cs | 10 +--- ....Configuration.EnvironmentVariables.csproj | 2 + .../tests/EnvironmentVariablesTest.cs | 56 +++++++++++++++++-- 3 files changed, 55 insertions(+), 13 deletions(-) diff --git a/src/libraries/Microsoft.Extensions.Configuration.EnvironmentVariables/src/EnvironmentVariablesConfigurationProvider.cs b/src/libraries/Microsoft.Extensions.Configuration.EnvironmentVariables/src/EnvironmentVariablesConfigurationProvider.cs index 5ea29116ca2d9b..55f5290cd13da9 100644 --- a/src/libraries/Microsoft.Extensions.Configuration.EnvironmentVariables/src/EnvironmentVariablesConfigurationProvider.cs +++ b/src/libraries/Microsoft.Extensions.Configuration.EnvironmentVariables/src/EnvironmentVariablesConfigurationProvider.cs @@ -78,17 +78,9 @@ internal void Load(IDictionary envVariables) { prefix = CustomPrefix; } - else if (key.StartsWith(_prefix, StringComparison.OrdinalIgnoreCase)) - { - // This prevents the prefix from being normalized. - // We can also do a fast path branch, I guess? No point in reallocating if the prefix is empty. - key = NormalizeKey(key.Substring(_prefix.Length)); - data[key] = entry.Value as string; - - continue; - } else { + AddIfPrefixed(data, NormalizeKey(key), (string?)entry.Value); continue; } diff --git a/src/libraries/Microsoft.Extensions.Configuration.EnvironmentVariables/src/Microsoft.Extensions.Configuration.EnvironmentVariables.csproj b/src/libraries/Microsoft.Extensions.Configuration.EnvironmentVariables/src/Microsoft.Extensions.Configuration.EnvironmentVariables.csproj index c12fa852d9b768..39446c4ca215cc 100644 --- a/src/libraries/Microsoft.Extensions.Configuration.EnvironmentVariables/src/Microsoft.Extensions.Configuration.EnvironmentVariables.csproj +++ b/src/libraries/Microsoft.Extensions.Configuration.EnvironmentVariables/src/Microsoft.Extensions.Configuration.EnvironmentVariables.csproj @@ -4,6 +4,8 @@ netstandard2.0;net461 true Environment variables configuration provider implementation for Microsoft.Extensions.Configuration. + true + 1 diff --git a/src/libraries/Microsoft.Extensions.Configuration.EnvironmentVariables/tests/EnvironmentVariablesTest.cs b/src/libraries/Microsoft.Extensions.Configuration.EnvironmentVariables/tests/EnvironmentVariablesTest.cs index c5b1030ee74a0b..1e98a5767096a0 100644 --- a/src/libraries/Microsoft.Extensions.Configuration.EnvironmentVariables/tests/EnvironmentVariablesTest.cs +++ b/src/libraries/Microsoft.Extensions.Configuration.EnvironmentVariables/tests/EnvironmentVariablesTest.cs @@ -166,7 +166,7 @@ public void ReplaceDoubleUnderscoreInEnvironmentVariablesButNotPrefix() envConfigSrc.Load(dict); - Assert.Equal("connection", envConfigSrc.Get("data:ConnectionString")); + Assert.Throws(() => envConfigSrc.Get("data:ConnectionString")); } [Fact] @@ -176,7 +176,7 @@ public void ReplaceDoubleUnderscoreInEnvironmentVariablesButNotInAnomalousPrefix { {"_____EXPERIMENTAL__data__ConnectionString", "connection"} }; - var envConfigSrc = new EnvironmentVariablesConfigurationProvider("_____EXPERIMENTAL__"); + var envConfigSrc = new EnvironmentVariablesConfigurationProvider("::_EXPERIMENTAL:"); envConfigSrc.Load(dict); @@ -194,7 +194,7 @@ public void ReplaceDoubleUnderscoreInEnvironmentVariablesWithDuplicatedPrefix() envConfigSrc.Load(dict); - Assert.Equal("connection", envConfigSrc.Get("test:ConnectionString")); + Assert.Throws(() => envConfigSrc.Get("test:ConnectionString")); } [Fact] @@ -205,7 +205,7 @@ public void PrefixPreventsLoadingSqlConnectionStrings() {"test__test__ConnectionString", "connection"}, {"SQLCONNSTR_db1", "connStr"} }; - var envConfigSrc = new EnvironmentVariablesConfigurationProvider("test__"); + var envConfigSrc = new EnvironmentVariablesConfigurationProvider("test:"); envConfigSrc.Load(dict); @@ -213,6 +213,54 @@ public void PrefixPreventsLoadingSqlConnectionStrings() Assert.Throws(() => envConfigSrc.Get("ConnectionStrings:db1_ProviderName")); } + public const string EnvironmentVariable = "Microsoft__Extensions__Configuration__EnvironmentVariables__Test__Foo"; + public class SettingsWithFoo + { + public string? Foo { get; set; } + } + + [Fact] + public void AddEnvironmentVariables_Bind_PrefixShouldNormalize() + { + try + { + Environment.SetEnvironmentVariable(EnvironmentVariable, "myFooValue"); + var configuration = new ConfigurationBuilder() + .AddEnvironmentVariables("Microsoft:Extensions:Configuration:EnvironmentVariables:Test:") + .Build(); + + var settingsWithFoo = new SettingsWithFoo(); + configuration.Bind(settingsWithFoo); + + Assert.Equal("myFooValue", settingsWithFoo.Foo); + } + finally + { + Environment.SetEnvironmentVariable(EnvironmentVariable, null); + } + } + + [Fact] + public void AddEnvironmentVariables_UsingDoubleUnderscores_Bind_PrefixWontNormalize() + { + try + { + Environment.SetEnvironmentVariable(EnvironmentVariable, "myFooValue"); + var configuration = new ConfigurationBuilder() + .AddEnvironmentVariables("Microsoft__Extensions__Configuration__EnvironmentVariables__Test__") + .Build(); + + var settingsWithFoo = new SettingsWithFoo(); + configuration.Bind(settingsWithFoo); + + Assert.Null(settingsWithFoo.Foo); + } + finally + { + Environment.SetEnvironmentVariable(EnvironmentVariable, null); + } + } + [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsThreadingSupported))] public void BindingDoesNotThrowIfReloadedDuringBinding() { From a8b733b0813010771392225156be2baa3ff9ada1 Mon Sep 17 00:00:00 2001 From: Eric Erhardt Date: Fri, 7 Jan 2022 12:45:06 -0600 Subject: [PATCH 20/39] [release/6.0] Ensure UserSecretsIdAttribute is added to assembly (#63423) * [release/6.0] Ensure UserSecretsIdAttribute is added to assembly When Microsoft.Extensions.Configuration.UserSecrets is referenced transitively, for example through Extensions.Hosting, the UserSecretsIdAttribute isn't getting added to the built assembly. This is because the logic is contained in a `build` folder which is excluded by the dependency from Extensions.Hosting. The fix is to move the logic to `buildTransitive`, so it gets picked up by NuGet. Note, we also need to service the Extensions.Hosting package so it references the new UserSecrets version by default. Fix #63246 * Add a binding redirect in order to fix test failure on netfx. --- .../Microsoft.Extensions.Configuration.UserSecrets.csproj | 4 ++-- .../Microsoft.Extensions.Configuration.UserSecrets.props | 0 .../Microsoft.Extensions.Configuration.UserSecrets.targets | 0 .../Microsoft.Extensions.Hosting/Directory.Build.props | 1 + .../src/Microsoft.Extensions.Hosting.csproj | 4 +++- .../Microsoft.Extensions.Hosting.Unit.Tests.csproj | 6 ++++++ 6 files changed, 12 insertions(+), 3 deletions(-) rename src/libraries/Microsoft.Extensions.Configuration.UserSecrets/src/{build => buildTransitive}/netstandard2.0/Microsoft.Extensions.Configuration.UserSecrets.props (100%) rename src/libraries/Microsoft.Extensions.Configuration.UserSecrets/src/{build => buildTransitive}/netstandard2.0/Microsoft.Extensions.Configuration.UserSecrets.targets (100%) diff --git a/src/libraries/Microsoft.Extensions.Configuration.UserSecrets/src/Microsoft.Extensions.Configuration.UserSecrets.csproj b/src/libraries/Microsoft.Extensions.Configuration.UserSecrets/src/Microsoft.Extensions.Configuration.UserSecrets.csproj index 3fda47281fe0f4..659217f32d2152 100644 --- a/src/libraries/Microsoft.Extensions.Configuration.UserSecrets/src/Microsoft.Extensions.Configuration.UserSecrets.csproj +++ b/src/libraries/Microsoft.Extensions.Configuration.UserSecrets/src/Microsoft.Extensions.Configuration.UserSecrets.csproj @@ -16,8 +16,8 @@ - - + + diff --git a/src/libraries/Microsoft.Extensions.Configuration.UserSecrets/src/build/netstandard2.0/Microsoft.Extensions.Configuration.UserSecrets.props b/src/libraries/Microsoft.Extensions.Configuration.UserSecrets/src/buildTransitive/netstandard2.0/Microsoft.Extensions.Configuration.UserSecrets.props similarity index 100% rename from src/libraries/Microsoft.Extensions.Configuration.UserSecrets/src/build/netstandard2.0/Microsoft.Extensions.Configuration.UserSecrets.props rename to src/libraries/Microsoft.Extensions.Configuration.UserSecrets/src/buildTransitive/netstandard2.0/Microsoft.Extensions.Configuration.UserSecrets.props diff --git a/src/libraries/Microsoft.Extensions.Configuration.UserSecrets/src/build/netstandard2.0/Microsoft.Extensions.Configuration.UserSecrets.targets b/src/libraries/Microsoft.Extensions.Configuration.UserSecrets/src/buildTransitive/netstandard2.0/Microsoft.Extensions.Configuration.UserSecrets.targets similarity index 100% rename from src/libraries/Microsoft.Extensions.Configuration.UserSecrets/src/build/netstandard2.0/Microsoft.Extensions.Configuration.UserSecrets.targets rename to src/libraries/Microsoft.Extensions.Configuration.UserSecrets/src/buildTransitive/netstandard2.0/Microsoft.Extensions.Configuration.UserSecrets.targets diff --git a/src/libraries/Microsoft.Extensions.Hosting/Directory.Build.props b/src/libraries/Microsoft.Extensions.Hosting/Directory.Build.props index 43dc3a2640bbae..207017db91b3d1 100644 --- a/src/libraries/Microsoft.Extensions.Hosting/Directory.Build.props +++ b/src/libraries/Microsoft.Extensions.Hosting/Directory.Build.props @@ -2,5 +2,6 @@ true + 1 diff --git a/src/libraries/Microsoft.Extensions.Hosting/src/Microsoft.Extensions.Hosting.csproj b/src/libraries/Microsoft.Extensions.Hosting/src/Microsoft.Extensions.Hosting.csproj index b868be57845e0c..f80e9513f42411 100644 --- a/src/libraries/Microsoft.Extensions.Hosting/src/Microsoft.Extensions.Hosting.csproj +++ b/src/libraries/Microsoft.Extensions.Hosting/src/Microsoft.Extensions.Hosting.csproj @@ -1,4 +1,4 @@ - + $(NetCoreAppCurrent);netstandard2.0;netstandard2.1;net461 @@ -6,6 +6,8 @@ Hosting and startup infrastructures for applications. false + true + diff --git a/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/Microsoft.Extensions.Hosting.Unit.Tests.csproj b/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/Microsoft.Extensions.Hosting.Unit.Tests.csproj index ee0c814c41b3f0..babb650155cbfb 100644 --- a/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/Microsoft.Extensions.Hosting.Unit.Tests.csproj +++ b/src/libraries/Microsoft.Extensions.Hosting/tests/UnitTests/Microsoft.Extensions.Hosting.Unit.Tests.csproj @@ -18,4 +18,10 @@ + + + + + From d575a6ff9093b577d716efd4a8e5aaaad9d645b9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 7 Jan 2022 11:00:15 -0800 Subject: [PATCH 21/39] [release/6.0] Add explicit null-check for tailcalls to VSD (#62769) * Add explicit null-check for tailcalls to VSD There is already a comment that this is necessary, but it is only being done for x86 tailcalls via jit helper. Do it for normal tailcalls to VSD as well. Fix #61486 * Revert cleanup to make potential backport easier Co-authored-by: Jakob Botsch Nielsen --- src/coreclr/jit/morph.cpp | 18 ++++---- .../JitBlue/Runtime_61486/Runtime_61486.cs | 45 +++++++++++++++++++ .../Runtime_61486/Runtime_61486.csproj | 9 ++++ 3 files changed, 63 insertions(+), 9 deletions(-) create mode 100644 src/tests/JIT/Regression/JitBlue/Runtime_61486/Runtime_61486.cs create mode 100644 src/tests/JIT/Regression/JitBlue/Runtime_61486/Runtime_61486.csproj diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index f6bc9a4bbe0598..ba5f450b616c9f 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -7896,6 +7896,15 @@ GenTree* Compiler::fgMorphPotentialTailCall(GenTreeCall* call) // Avoid potential extra work for the return (for example, vzeroupper) call->gtType = TYP_VOID; + // The runtime requires that we perform a null check on the `this` argument before + // tail calling to a virtual dispatch stub. This requirement is a consequence of limitations + // in the runtime's ability to map an AV to a NullReferenceException if + // the AV occurs in a dispatch stub that has unmanaged caller. + if (call->IsVirtualStub()) + { + call->gtFlags |= GTF_CALL_NULLCHECK; + } + // Do some target-specific transformations (before we process the args, // etc.) for the JIT helper case. if (tailCallViaJitHelper) @@ -8622,15 +8631,6 @@ void Compiler::fgMorphTailCallViaJitHelper(GenTreeCall* call) JITDUMP("fgMorphTailCallViaJitHelper (before):\n"); DISPTREE(call); - // The runtime requires that we perform a null check on the `this` argument before - // tail calling to a virtual dispatch stub. This requirement is a consequence of limitations - // in the runtime's ability to map an AV to a NullReferenceException if - // the AV occurs in a dispatch stub that has unmanaged caller. - if (call->IsVirtualStub()) - { - call->gtFlags |= GTF_CALL_NULLCHECK; - } - // For the helper-assisted tail calls, we need to push all the arguments // into a single list, and then add a few extra at the beginning or end. // diff --git a/src/tests/JIT/Regression/JitBlue/Runtime_61486/Runtime_61486.cs b/src/tests/JIT/Regression/JitBlue/Runtime_61486/Runtime_61486.cs new file mode 100644 index 00000000000000..3285ee9cc556a9 --- /dev/null +++ b/src/tests/JIT/Regression/JitBlue/Runtime_61486/Runtime_61486.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Reflection; +using System.Runtime.CompilerServices; + +public class Runtime_61486 +{ + public static int Main() + { + var my = new My(new My(null)); + var m = my.GetType().GetMethod("M"); + try + { + m.Invoke(my, null); + return -1; + } + catch (TargetInvocationException ex) when (ex.InnerException is NullReferenceException) + { + return 100; + } + } + + public interface IFace + { + void M(); + } + + public class My : IFace + { + private IFace _face; + + public My(IFace face) + { + _face = face; + } + + // We cannot handle a null ref inside a VSD if the caller is not + // managed frame. This test is verifying that JIT null checks ahead of + // time in this case. + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + public void M() => _face.M(); + } +} diff --git a/src/tests/JIT/Regression/JitBlue/Runtime_61486/Runtime_61486.csproj b/src/tests/JIT/Regression/JitBlue/Runtime_61486/Runtime_61486.csproj new file mode 100644 index 00000000000000..6946bed81bfd5b --- /dev/null +++ b/src/tests/JIT/Regression/JitBlue/Runtime_61486/Runtime_61486.csproj @@ -0,0 +1,9 @@ + + + Exe + True + + + + + From dc56b08106d9d4ee986c7ddf04deceb9bccea275 Mon Sep 17 00:00:00 2001 From: Dan Moseley Date: Fri, 7 Jan 2022 16:33:19 -0700 Subject: [PATCH 22/39] [release/6.0] Port test-only test fixes (#63507) * Outerloop ports * Port EFS test fix * Partially disable test (#59760) * fix build --- .../System/PlatformDetection.Windows.cs | 2 + .../System/WindowsIdentityFixture.cs | 7 ++ .../Enumeration/SpecialDirectoryTests.cs | 9 +- .../tests/File/EncryptDecrypt.Windows.cs | 85 +++++++++++++++++++ .../tests/File/EncryptDecrypt.cs | 40 +++++++-- ...stem.IO.FileSystem.Net5Compat.Tests.csproj | 1 + .../tests/System.IO.FileSystem.Tests.csproj | 1 + .../tests/SerialPort/DosDevices.cs | 9 +- .../FunctionalTests/ImpersonatedAuthTests.cs | 5 +- .../tests/FunctionalTests/PingTest.cs | 4 +- ...owsIdentityImpersonatedTests.netcoreapp.cs | 6 +- 11 files changed, 154 insertions(+), 15 deletions(-) create mode 100644 src/libraries/System.IO.FileSystem/tests/File/EncryptDecrypt.Windows.cs diff --git a/src/libraries/Common/tests/TestUtilities/System/PlatformDetection.Windows.cs b/src/libraries/Common/tests/TestUtilities/System/PlatformDetection.Windows.cs index d5b2c36e961e0e..cf5e4b86b87b84 100644 --- a/src/libraries/Common/tests/TestUtilities/System/PlatformDetection.Windows.cs +++ b/src/libraries/Common/tests/TestUtilities/System/PlatformDetection.Windows.cs @@ -234,6 +234,8 @@ public static bool IsInAppContainer } } + public static bool CanRunImpersonatedTests => PlatformDetection.IsNotWindowsNanoServer && PlatformDetection.IsWindowsAndElevated; + private static int s_isWindowsElevated = -1; public static bool IsWindowsAndElevated { diff --git a/src/libraries/Common/tests/TestUtilities/System/WindowsIdentityFixture.cs b/src/libraries/Common/tests/TestUtilities/System/WindowsIdentityFixture.cs index 360fbe68bb7ac4..b0355f923295e4 100644 --- a/src/libraries/Common/tests/TestUtilities/System/WindowsIdentityFixture.cs +++ b/src/libraries/Common/tests/TestUtilities/System/WindowsIdentityFixture.cs @@ -10,6 +10,7 @@ using System.Security.Principal; using System.Threading.Tasks; using Microsoft.Win32.SafeHandles; +using Xunit; namespace System { @@ -37,6 +38,8 @@ public sealed class WindowsTestAccount : IDisposable public WindowsTestAccount(string userName) { + Assert.True(PlatformDetection.IsWindowsAndElevated); + _userName = userName; CreateUser(); } @@ -77,6 +80,10 @@ private void CreateUser() throw new Win32Exception((int)result); } } + else if (result != 0) + { + throw new Win32Exception((int)result); + } const int LOGON32_PROVIDER_DEFAULT = 0; const int LOGON32_LOGON_INTERACTIVE = 2; diff --git a/src/libraries/System.IO.FileSystem/tests/Enumeration/SpecialDirectoryTests.cs b/src/libraries/System.IO.FileSystem/tests/Enumeration/SpecialDirectoryTests.cs index 39a747179481a8..12edab6601e803 100644 --- a/src/libraries/System.IO.FileSystem/tests/Enumeration/SpecialDirectoryTests.cs +++ b/src/libraries/System.IO.FileSystem/tests/Enumeration/SpecialDirectoryTests.cs @@ -60,7 +60,14 @@ public void SkippingHiddenFiles() { // Files that begin with periods are considered hidden on Unix string[] paths = GetNames(TestDirectory, new EnumerationOptions { ReturnSpecialDirectories = true, AttributesToSkip = 0 }); - Assert.Contains(".", paths); + + if (!PlatformDetection.IsWindows10Version22000OrGreater) + { + // Sometimes this is not returned - presumably an OS bug. + // This occurs often on Windows 11, very rarely otherwise. + Assert.Contains(".", paths); + } + Assert.Contains("..", paths); } } diff --git a/src/libraries/System.IO.FileSystem/tests/File/EncryptDecrypt.Windows.cs b/src/libraries/System.IO.FileSystem/tests/File/EncryptDecrypt.Windows.cs new file mode 100644 index 00000000000000..015ba30b6ad7c1 --- /dev/null +++ b/src/libraries/System.IO.FileSystem/tests/File/EncryptDecrypt.Windows.cs @@ -0,0 +1,85 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.XUnitExtensions; +using System.Diagnostics; +using System.Diagnostics.Eventing.Reader; +using System.Linq; +using System.Security; +using System.ServiceProcess; +using Xunit; +using Xunit.Abstractions; + +namespace System.IO.Tests +{ + public partial class EncryptDecrypt + { + partial void EnsureEFSServiceStarted() + { + try + { + using var sc = new ServiceController("EFS"); + _output.WriteLine($"EFS service is: {sc.Status}"); + if (sc.Status != ServiceControllerStatus.Running) + { + _output.WriteLine("Trying to start EFS service"); + sc.Start(); + _output.WriteLine($"EFS service is now: {sc.Status}"); + } + } + catch (Exception e) + { + _output.WriteLine(e.ToString()); + } + } + + partial void LogEFSDiagnostics() + { + int hours = 1; // how many hours to look backwards + string query = @$" + + + + + + + *[System[TimeCreated[timediff(@SystemTime) >= {hours * 60 * 60 * 1000L}]]] + + + "; + + var eventQuery = new EventLogQuery("System", PathType.LogName, query); + + using var eventReader = new EventLogReader(eventQuery); + + EventRecord record = eventReader.ReadEvent(); + var garbage = new string[] { "Background Intelligent", "Intel", "Defender", "Intune", "BITS", "NetBT"}; + + _output.WriteLine("===== Dumping recent relevant events: ====="); + while (record != null) + { + string description = ""; + try + { + description = record.FormatDescription(); + } + catch (EventLogException) { } + + if (!garbage.Any(term => description.Contains(term, StringComparison.OrdinalIgnoreCase))) + { + _output.WriteLine($"{record.TimeCreated} {record.ProviderName} [{record.LevelDisplayName} {record.Id}] {description.Replace("\r\n", " ")}"); + } + + record = eventReader.ReadEvent(); + } + + _output.WriteLine("==== Finished dumping ====="); + } + } +} diff --git a/src/libraries/System.IO.FileSystem/tests/File/EncryptDecrypt.cs b/src/libraries/System.IO.FileSystem/tests/File/EncryptDecrypt.cs index 984afef972bc41..1dbac453e48297 100644 --- a/src/libraries/System.IO.FileSystem/tests/File/EncryptDecrypt.cs +++ b/src/libraries/System.IO.FileSystem/tests/File/EncryptDecrypt.cs @@ -1,17 +1,28 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.DotNet.XUnitExtensions; using System.Diagnostics; +using System.Diagnostics.Eventing.Reader; using System.Security; +using System.ServiceProcess; using Xunit; +using Xunit.Abstractions; namespace System.IO.Tests { [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] - public class EncryptDecrypt : FileSystemTest + public partial class EncryptDecrypt : FileSystemTest { + private readonly ITestOutputHelper _output; + + public EncryptDecrypt(ITestOutputHelper output) + { + _output = output; + } + [Fact] - public static void NullArg_ThrowsException() + public void NullArg_ThrowsException() { AssertExtensions.Throws("path", () => File.Encrypt(null)); AssertExtensions.Throws("path", () => File.Decrypt(null)); @@ -19,7 +30,7 @@ public static void NullArg_ThrowsException() [SkipOnTargetFramework(TargetFrameworkMonikers.Netcoreapp)] [Fact] - public static void EncryptDecrypt_NotSupported() + public void EncryptDecrypt_NotSupported() { Assert.Throws(() => File.Encrypt("path")); Assert.Throws(() => File.Decrypt("path")); @@ -29,7 +40,8 @@ public static void EncryptDecrypt_NotSupported() // because EFS (Encrypted File System), its underlying technology, is not available on these operating systems. [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoServer), nameof(PlatformDetection.IsNotWindowsHomeEdition))] [PlatformSpecific(TestPlatforms.Windows)] - public static void EncryptDecrypt_Read() + [OuterLoop] // Occasional failures: https://github.com/dotnet/runtime/issues/12339 + public void EncryptDecrypt_Read() { string tmpFileName = Path.GetTempFileName(); string textContentToEncrypt = "Content to encrypt"; @@ -39,14 +51,26 @@ public static void EncryptDecrypt_Read() string fileContentRead = File.ReadAllText(tmpFileName); Assert.Equal(textContentToEncrypt, fileContentRead); + EnsureEFSServiceStarted(); + try { File.Encrypt(tmpFileName); } - catch (IOException e) when (e.HResult == unchecked((int)0x80070490)) + catch (IOException e) when (e.HResult == unchecked((int)0x80070490) || + e.HResult == unchecked((int)0x80071776) || + e.HResult == unchecked((int)0x800701AE)) { // Ignore ERROR_NOT_FOUND 1168 (0x490). It is reported when EFS is disabled by domain policy. - return; + // Ignore ERROR_NO_USER_KEYS (0x1776). This occurs when no user key exists to encrypt with. + // Ignore ERROR_ENCRYPTION_DISABLED (0x1AE). This occurs when EFS is disabled by group policy on the volume. + throw new SkipTestException($"Encrypt not available. Error 0x{e.HResult:X}"); + } + catch (IOException e) + { + _output.WriteLine($"Encrypt failed with {e.Message} 0x{e.HResult:X}"); + LogEFSDiagnostics(); + throw; } Assert.Equal(fileContentRead, File.ReadAllText(tmpFileName)); @@ -61,5 +85,9 @@ public static void EncryptDecrypt_Read() File.Delete(tmpFileName); } } + + partial void EnsureEFSServiceStarted(); // no-op on Unix + + partial void LogEFSDiagnostics(); // no-op on Unix currently } } diff --git a/src/libraries/System.IO.FileSystem/tests/Net5CompatTests/System.IO.FileSystem.Net5Compat.Tests.csproj b/src/libraries/System.IO.FileSystem/tests/Net5CompatTests/System.IO.FileSystem.Net5Compat.Tests.csproj index 438e076c3e778e..385c3afb569c49 100644 --- a/src/libraries/System.IO.FileSystem/tests/Net5CompatTests/System.IO.FileSystem.Net5Compat.Tests.csproj +++ b/src/libraries/System.IO.FileSystem/tests/Net5CompatTests/System.IO.FileSystem.Net5Compat.Tests.csproj @@ -32,6 +32,7 @@ + diff --git a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj index 1ffb30f887c246..4c8fa1c23be5b8 100644 --- a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj +++ b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj @@ -77,6 +77,7 @@ + diff --git a/src/libraries/System.IO.Ports/tests/SerialPort/DosDevices.cs b/src/libraries/System.IO.Ports/tests/SerialPort/DosDevices.cs index 3fe9493c0a30e5..2d74327f97eb0d 100644 --- a/src/libraries/System.IO.Ports/tests/SerialPort/DosDevices.cs +++ b/src/libraries/System.IO.Ports/tests/SerialPort/DosDevices.cs @@ -3,6 +3,7 @@ using System.Collections; using System.Collections.Generic; +using System.ComponentModel; using System.Runtime.InteropServices; namespace System.IO.Ports.Tests @@ -127,14 +128,20 @@ private static char[] CallQueryDosDevice(string name, out int dataSize) buffer = new char[buffer.Length * 2]; dataSize = QueryDosDevice(null, buffer, buffer.Length); } + else if (lastError == ERROR_ACCESS_DENIED) // Access denied eg for "MSSECFLTSYS" - just skip + { + dataSize = 0; + break; + } else { - throw new Exception("Unknown Win32 Error: " + lastError); + throw new Exception($"Error {lastError} calling QueryDosDevice for '{name}' with buffer size {buffer.Length}. {new Win32Exception((int)lastError).Message}"); } } return buffer; } + public const int ERROR_ACCESS_DENIED = 5; public const int ERROR_INSUFFICIENT_BUFFER = 122; public const int ERROR_MORE_DATA = 234; diff --git a/src/libraries/System.Net.Http/tests/FunctionalTests/ImpersonatedAuthTests.cs b/src/libraries/System.Net.Http/tests/FunctionalTests/ImpersonatedAuthTests.cs index 6b56d3590d9ad0..32f44d5e00f705 100644 --- a/src/libraries/System.Net.Http/tests/FunctionalTests/ImpersonatedAuthTests.cs +++ b/src/libraries/System.Net.Http/tests/FunctionalTests/ImpersonatedAuthTests.cs @@ -14,7 +14,6 @@ namespace System.Net.Http.Functional.Tests { public class ImpersonatedAuthTests: IClassFixture { - public static bool CanRunImpersonatedTests = PlatformDetection.IsWindows && PlatformDetection.IsNotWindowsNanoServer; private readonly WindowsIdentityFixture _fixture; private readonly ITestOutputHelper _output; @@ -28,11 +27,11 @@ public ImpersonatedAuthTests(WindowsIdentityFixture windowsIdentityFixture, ITe } [OuterLoop] - [ConditionalTheory(nameof(CanRunImpersonatedTests))] + [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.CanRunImpersonatedTests))] [InlineData(true)] [InlineData(false)] [PlatformSpecific(TestPlatforms.Windows)] - public async Task DefaultHandler_ImpersonificatedUser_Success(bool useNtlm) + public async Task DefaultHandler_ImpersonatedUser_Success(bool useNtlm) { await LoopbackServer.CreateClientAndServerAsync( async uri => diff --git a/src/libraries/System.Net.Ping/tests/FunctionalTests/PingTest.cs b/src/libraries/System.Net.Ping/tests/FunctionalTests/PingTest.cs index b206e1e0f165d9..32ef250d8657eb 100644 --- a/src/libraries/System.Net.Ping/tests/FunctionalTests/PingTest.cs +++ b/src/libraries/System.Net.Ping/tests/FunctionalTests/PingTest.cs @@ -878,7 +878,9 @@ public async Task SendPingToExternalHostWithLowTtlTest() options.Ttl = 1; // This should always fail unless host is one IP hop away. pingReply = await ping.SendPingAsync(host, TestSettings.PingTimeout, TestSettings.PayloadAsBytesShort, options); - Assert.NotEqual(IPStatus.Success, pingReply.Status); + + Assert.True(pingReply.Status == IPStatus.TimeExceeded || pingReply.Status == IPStatus.TtlExpired); + Assert.NotEqual(IPAddress.Any, pingReply.Address); } [Fact] diff --git a/src/libraries/System.Security.Principal.Windows/tests/WindowsIdentityImpersonatedTests.netcoreapp.cs b/src/libraries/System.Security.Principal.Windows/tests/WindowsIdentityImpersonatedTests.netcoreapp.cs index 4bfb059d708c8c..0a7c1745ed9f42 100644 --- a/src/libraries/System.Security.Principal.Windows/tests/WindowsIdentityImpersonatedTests.netcoreapp.cs +++ b/src/libraries/System.Security.Principal.Windows/tests/WindowsIdentityImpersonatedTests.netcoreapp.cs @@ -26,7 +26,7 @@ public WindowsIdentityImpersonatedTests(WindowsIdentityFixture windowsIdentityFi Assert.False(string.IsNullOrEmpty(_fixture.TestAccount.AccountName)); } - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoServer))] + [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.CanRunImpersonatedTests))] [OuterLoop] public async Task RunImpersonatedAsync_TaskAndTaskOfT() { @@ -60,7 +60,7 @@ void Asserts(WindowsIdentity currentWindowsIdentity) } } - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoServer))] + [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.CanRunImpersonatedTests))] [OuterLoop] public void RunImpersonated_NameResolution() { @@ -78,7 +78,7 @@ public void RunImpersonated_NameResolution() }); } - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoServer))] + [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.CanRunImpersonatedTests))] [OuterLoop] public async Task RunImpersonatedAsync_NameResolution() { From abe7d08647bcbf0f15bc23a1da3e326a35cc2317 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 7 Jan 2022 17:01:06 -0800 Subject: [PATCH 23/39] [release/6.0] Fix System.Object serialization with custom number handling (#62193) * Fix System.Object serialization with custom number handling * fix typos * Address feedback Co-authored-by: Eirik Tsarpalis Co-authored-by: Eric StJohn --- .../Converters/Value/ObjectConverter.cs | 15 --------------- .../Serialization/Object.ReadTests.cs | 16 ++++++++++++++++ .../Serialization/Object.WriteTests.cs | 8 ++++++++ 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/ObjectConverter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/ObjectConverter.cs index ddb2a899947b64..7ce4a2cb706b55 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/ObjectConverter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/ObjectConverter.cs @@ -7,11 +7,6 @@ namespace System.Text.Json.Serialization.Converters { internal sealed class ObjectConverter : JsonConverter { - public ObjectConverter() - { - IsInternalConverterForNumberType = true; - } - public override object? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { if (options.UnknownTypeHandling == JsonUnknownTypeHandling.JsonElement) @@ -50,15 +45,5 @@ internal override void WriteAsPropertyNameCore(Utf8JsonWriter writer, object? va runtimeConverter.WriteAsPropertyNameCoreAsObject(writer, value, options, isWritingExtensionDataProperty); } - - internal override object? ReadNumberWithCustomHandling(ref Utf8JsonReader reader, JsonNumberHandling handling, JsonSerializerOptions options) - { - if (options.UnknownTypeHandling == JsonUnknownTypeHandling.JsonElement) - { - return JsonElement.ParseValue(ref reader); - } - - return JsonNodeConverter.Instance.Read(ref reader, typeof(object), options); - } } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Object.ReadTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Object.ReadTests.cs index 6a36e96d3818b9..c8a745398af275 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Object.ReadTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Object.ReadTests.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Text.Json.Nodes; using Xunit; namespace System.Text.Json.Serialization.Tests @@ -655,5 +656,20 @@ public static void TooLittleJsonFails(string json) Assert.Equal(0, reader.BytesConsumed); } + + [Theory] + [InlineData(JsonUnknownTypeHandling.JsonElement, typeof(JsonElement))] + [InlineData(JsonUnknownTypeHandling.JsonNode, typeof(JsonNode))] + public static void ReadSystemObjectWithNumberHandling(JsonUnknownTypeHandling unknownTypeHandling, Type expectedType) + { + var options = new JsonSerializerOptions + { + NumberHandling = JsonNumberHandling.AllowReadingFromString, + UnknownTypeHandling = unknownTypeHandling + }; + + object result = JsonSerializer.Deserialize(@"{ ""key"" : ""42"" }", options); + Assert.IsAssignableFrom(expectedType, result); + } } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Object.WriteTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Object.WriteTests.cs index d4e551d6d01b25..c569770e3027e1 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Object.WriteTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Object.WriteTests.cs @@ -138,5 +138,13 @@ public static void EscapingShouldntStackOverflow() Assert.Equal("{\"name\":\"\u6D4B\u8A6611\"}", result); } + + [Fact] + public static void WriteSystemObjectWithNumberHandling() + { + var options = new JsonSerializerOptions { NumberHandling = JsonNumberHandling.AllowReadingFromString }; + string result = JsonSerializer.Serialize(new object(), options); + Assert.Equal("{}", result); + } } } From 039ab90e66ee4409b13543fe9a553a1f41e89958 Mon Sep 17 00:00:00 2001 From: Layomi Akinrinade Date: Fri, 7 Jan 2022 22:32:08 -0500 Subject: [PATCH 24/39] Fix bugs with generation for ctor param default values (#62798) (#63493) * Fix bugs with generation for ctor param default values * Address review feedback * Address feedback --- .../System.Text.Json/Common/JsonConstants.cs | 4 + .../gen/JsonSourceGenerator.Emitter.cs | 94 +++++++++--- .../gen/JsonSourceGenerator.Parser.cs | 3 + .../gen/SourceGenerationSpec.cs | 5 + .../Utf8JsonWriter.WriteValues.Double.cs | 4 +- .../Utf8JsonWriter.WriteValues.Float.cs | 4 +- .../ConstructorTests.ParameterMatching.cs | 138 ++++++++++++++++++ .../Serialization/ConstructorTests.cs | 2 + 8 files changed, 224 insertions(+), 30 deletions(-) diff --git a/src/libraries/System.Text.Json/Common/JsonConstants.cs b/src/libraries/System.Text.Json/Common/JsonConstants.cs index a97b11cf15bf2d..0d121d9c6ffbc8 100644 --- a/src/libraries/System.Text.Json/Common/JsonConstants.cs +++ b/src/libraries/System.Text.Json/Common/JsonConstants.cs @@ -7,5 +7,9 @@ internal static partial class JsonConstants { // The maximum number of parameters a constructor can have where it can be supported by the serializer. public const int MaxParameterCount = 64; + + // Standard format for double and single on non-inbox frameworks. + public const string DoubleFormatString = "G17"; + public const string SingleFormatString = "G9"; } } diff --git a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs index 087f3e2dc4095c..7cfda7f6a12be7 100644 --- a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs +++ b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs @@ -4,7 +4,9 @@ using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Globalization; using System.Reflection; +using System.Reflection.Metadata; using System.Text.Json; using System.Text.Json.Reflection; using System.Text.Json.Serialization; @@ -759,17 +761,17 @@ private string GeneratePropMetadataInitFunc(TypeGenerationSpec typeGenerationSpe sb.Append($@" {JsonPropertyInfoValuesTypeRef}<{memberTypeCompilableName}> {infoVarName} = new {JsonPropertyInfoValuesTypeRef}<{memberTypeCompilableName}>() {{ - IsProperty = {ToCSharpKeyword(memberMetadata.IsProperty)}, - IsPublic = {ToCSharpKeyword(memberMetadata.IsPublic)}, - IsVirtual = {ToCSharpKeyword(memberMetadata.IsVirtual)}, + IsProperty = {FormatBool(memberMetadata.IsProperty)}, + IsPublic = {FormatBool(memberMetadata.IsPublic)}, + IsVirtual = {FormatBool(memberMetadata.IsVirtual)}, DeclaringType = typeof({memberMetadata.DeclaringTypeRef}), PropertyTypeInfo = {memberTypeFriendlyName}, Converter = {converterValue}, Getter = {getterValue}, Setter = {setterValue}, IgnoreCondition = {ignoreConditionNamedArg}, - HasJsonInclude = {ToCSharpKeyword(memberMetadata.HasJsonInclude)}, - IsExtensionData = {ToCSharpKeyword(memberMetadata.IsExtensionData)}, + HasJsonInclude = {FormatBool(memberMetadata.HasJsonInclude)}, + IsExtensionData = {FormatBool(memberMetadata.IsExtensionData)}, NumberHandling = {GetNumberHandlingAsStr(memberMetadata.NumberHandling)}, PropertyName = ""{clrPropertyName}"", JsonPropertyName = {jsonPropertyNameValue} @@ -805,11 +807,11 @@ private string GenerateCtorParamMetadataInitFunc(TypeGenerationSpec typeGenerati for (int i = 0; i < paramCount; i++) { ParameterInfo reflectionInfo = parameters[i].ParameterInfo; - - string parameterTypeRef = reflectionInfo.ParameterType.GetCompilableName(); + Type parameterType = reflectionInfo.ParameterType; + string parameterTypeRef = parameterType.GetCompilableName(); object? defaultValue = reflectionInfo.GetDefaultValue(); - string defaultValueAsStr = GetParamDefaultValueAsString(defaultValue, parameterTypeRef); + string defaultValueAsStr = GetParamDefaultValueAsString(defaultValue, parameterType, parameterTypeRef); sb.Append(@$" {InfoVarName} = new() @@ -817,7 +819,7 @@ private string GenerateCtorParamMetadataInitFunc(TypeGenerationSpec typeGenerati Name = ""{reflectionInfo.Name!}"", ParameterType = typeof({parameterTypeRef}), Position = {reflectionInfo.Position}, - HasDefaultValue = {ToCSharpKeyword(reflectionInfo.HasDefaultValue)}, + HasDefaultValue = {FormatBool(reflectionInfo.HasDefaultValue)}, DefaultValue = {defaultValueAsStr} }}; {parametersVarName}[{i}] = {InfoVarName}; @@ -1192,10 +1194,10 @@ private string GetLogicForDefaultSerializerOptionsInit() private static {JsonSerializerOptionsTypeRef} {DefaultOptionsStaticVarName} {{ get; }} = new {JsonSerializerOptionsTypeRef}() {{ DefaultIgnoreCondition = {JsonIgnoreConditionTypeRef}.{options.DefaultIgnoreCondition}, - IgnoreReadOnlyFields = {ToCSharpKeyword(options.IgnoreReadOnlyFields)}, - IgnoreReadOnlyProperties = {ToCSharpKeyword(options.IgnoreReadOnlyProperties)}, - IncludeFields = {ToCSharpKeyword(options.IncludeFields)}, - WriteIndented = {ToCSharpKeyword(options.WriteIndented)},{namingPolicyInit} + IgnoreReadOnlyFields = {FormatBool(options.IgnoreReadOnlyFields)}, + IgnoreReadOnlyProperties = {FormatBool(options.IgnoreReadOnlyProperties)}, + IncludeFields = {FormatBool(options.IncludeFields)}, + WriteIndented = {FormatBool(options.WriteIndented)},{namingPolicyInit} }};"; } @@ -1316,20 +1318,64 @@ private static string GetNumberHandlingAsStr(JsonNumberHandling? numberHandling) : "default"; private static string GetCreateValueInfoMethodRef(string typeCompilableName) => $"{CreateValueInfoMethodName}<{typeCompilableName}>"; - } - private static string ToCSharpKeyword(bool value) => value.ToString().ToLowerInvariant(); + private static string FormatBool(bool value) => value ? "true" : "false"; - private static string GetParamDefaultValueAsString(object? value, string objectTypeAsStr) - { - switch (value) + private string GetParamDefaultValueAsString(object? value, Type type, string typeRef) { - case null: - return $"default({objectTypeAsStr})"; - case bool boolVal: - return ToCSharpKeyword(boolVal); - default: - return value!.ToString(); + if (value == null) + { + return $"default({typeRef})"; + } + + if (type.IsEnum) + { + // Roslyn gives us an instance of the underlying type, which is numerical. +#if DEBUG + Type runtimeType = _generationSpec.MetadataLoadContext.Resolve(value.GetType()); + Debug.Assert(_generationSpec.IsNumberType(runtimeType)); +#endif + + // Return the numeric value. + return FormatNumber(); + } + + switch (value) + { + case string @string: + return SymbolDisplay.FormatLiteral(@string, quote: true); ; + case char @char: + return SymbolDisplay.FormatLiteral(@char, quote: true); + case double.NegativeInfinity: + return "double.NegativeInfinity"; + case double.PositiveInfinity: + return "double.PositiveInfinity"; + case double.NaN: + return "double.NaN"; + case double @double: + return $"({typeRef})({@double.ToString(JsonConstants.DoubleFormatString, CultureInfo.InvariantCulture)})"; + case float.NegativeInfinity: + return "float.NegativeInfinity"; + case float.PositiveInfinity: + return "float.PositiveInfinity"; + case float.NaN: + return "float.NaN"; + case float @float: + return $"({typeRef})({@float.ToString(JsonConstants.SingleFormatString, CultureInfo.InvariantCulture)})"; + case decimal.MaxValue: + return "decimal.MaxValue"; + case decimal.MinValue: + return "decimal.MinValue"; + case decimal @decimal: + return @decimal.ToString(CultureInfo.InvariantCulture); + case bool @bool: + return FormatBool(@bool); + default: + // Assume this is a number. + return FormatNumber(); + } + + string FormatNumber() => $"({typeRef})({Convert.ToString(value, CultureInfo.InvariantCulture)})"; } } } diff --git a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs index fd190452e294f8..bc4b62669dd102 100644 --- a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs +++ b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs @@ -375,6 +375,9 @@ public Parser(Compilation compilation, in JsonSourceGenerationContext sourceGene GuidType = _guidType, StringType = _stringType, NumberTypes = _numberTypes, +#if DEBUG + MetadataLoadContext = _metadataLoadContext, +#endif }; } diff --git a/src/libraries/System.Text.Json/gen/SourceGenerationSpec.cs b/src/libraries/System.Text.Json/gen/SourceGenerationSpec.cs index 0019dc524955aa..09865cd403c5fa 100644 --- a/src/libraries/System.Text.Json/gen/SourceGenerationSpec.cs +++ b/src/libraries/System.Text.Json/gen/SourceGenerationSpec.cs @@ -4,6 +4,8 @@ using System; using System.Collections.Generic; using System.Text; +using System.Text.Json.Reflection; +using Microsoft.CodeAnalysis; namespace System.Text.Json.SourceGeneration { @@ -11,6 +13,9 @@ internal sealed class SourceGenerationSpec { public List ContextGenerationSpecList { get; init; } +#if DEBUG + public MetadataLoadContextInternal MetadataLoadContext { get; init; } +#endif public Type BooleanType { get; init; } public Type ByteArrayType { get; init; } public Type CharType { get; init; } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Double.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Double.cs index f8a46a31468ca9..bf16e32e9f8d57 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Double.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Double.cs @@ -109,9 +109,7 @@ private static bool TryFormatDouble(double value, Span destination, out in #if BUILDING_INBOX_LIBRARY return Utf8Formatter.TryFormat(value, destination, out bytesWritten); #else - const string FormatString = "G17"; - - string utf16Text = value.ToString(FormatString, CultureInfo.InvariantCulture); + string utf16Text = value.ToString(JsonConstants.DoubleFormatString, CultureInfo.InvariantCulture); // Copy the value to the destination, if it's large enough. diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Float.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Float.cs index 2f046d872f6972..067971e86b5af4 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Float.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Float.cs @@ -109,9 +109,7 @@ private static bool TryFormatSingle(float value, Span destination, out int #if BUILDING_INBOX_LIBRARY return Utf8Formatter.TryFormat(value, destination, out bytesWritten); #else - const string FormatString = "G9"; - - string utf16Text = value.ToString(FormatString, CultureInfo.InvariantCulture); + string utf16Text = value.ToString(JsonConstants.SingleFormatString, CultureInfo.InvariantCulture); // Copy the value to the destination, if it's large enough. diff --git a/src/libraries/System.Text.Json/tests/Common/ConstructorTests/ConstructorTests.ParameterMatching.cs b/src/libraries/System.Text.Json/tests/Common/ConstructorTests/ConstructorTests.ParameterMatching.cs index 93aff74c1b4d0b..d56a64a3e65c89 100644 --- a/src/libraries/System.Text.Json/tests/Common/ConstructorTests/ConstructorTests.ParameterMatching.cs +++ b/src/libraries/System.Text.Json/tests/Common/ConstructorTests/ConstructorTests.ParameterMatching.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Collections.Specialized; using System.Diagnostics.CodeAnalysis; +using System.Reflection; using System.Threading.Tasks; using Xunit; @@ -1322,5 +1323,142 @@ public class ClassWithIgnoredSameType public ClassWithIgnoredSameType(ClassWithIgnoredSameType prop) { } } + + public async Task TestClassWithDefaultCtorParams() + { + ClassWithDefaultCtorParams obj = new ClassWithDefaultCtorParams( + new Point_2D_Struct_WithAttribute(1, 2), + DayOfWeek.Sunday, + (DayOfWeek)(-2), + (DayOfWeek)19, + BindingFlags.CreateInstance | BindingFlags.FlattenHierarchy, + SampleEnumUInt32.MinZero, + "Hello world!", + null, + "xzy", + 'c', + ' ', + 23, + 4, + -40, + double.Epsilon, + 23, + 4, + -40, + float.MinValue, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10); + + string json = await JsonSerializerWrapperForString.SerializeWrapper(obj); + obj = await JsonSerializerWrapperForString.DeserializeWrapper(json); + JsonTestHelper.AssertJsonEqual(json, await JsonSerializerWrapperForString.SerializeWrapper(obj)); + } + + public class ClassWithDefaultCtorParams + { + public Point_2D_Struct_WithAttribute Struct { get; } + public DayOfWeek Enum1 { get; } + public DayOfWeek Enum2 { get; } + public DayOfWeek Enum3 { get; } + public BindingFlags Enum4 { get; } + public SampleEnumUInt32 Enum5 { get; } + public string Str1 { get; } + public string Str2 { get; } + public string Str3 { get; } + public char Char1 { get; } + public char Char2 { get; } + public double Double1 { get; } + public double Double2 { get; } + public double Double3 { get; } + public double Double4 { get; } + public double Double5 { get; } + public float Float1 { get; } + public float Float2 { get; } + public float Float3 { get; } + public float Float4 { get; } + public float Float5 { get; } + public byte Byte { get; } + public decimal Decimal1 { get; } + public decimal Decimal2 { get; } + public short Short { get; } + public sbyte Sbyte { get; } + public int Int { get; } + public long Long { get; } + public ushort UShort { get; } + public uint UInt { get; } + public ulong ULong { get; } + + public ClassWithDefaultCtorParams( + Point_2D_Struct_WithAttribute @struct = default, + DayOfWeek enum1 = default, + DayOfWeek enum2 = DayOfWeek.Sunday, + DayOfWeek enum3 = DayOfWeek.Sunday | DayOfWeek.Monday, + BindingFlags enum4 = BindingFlags.CreateInstance | BindingFlags.ExactBinding, + SampleEnumUInt32 enum5 = SampleEnumUInt32.MinZero, + string str1 = "abc", + string str2 = "", + string str3 = "\n\r⁉️\'\"\u200D\f\t\v\0\a\b\\\'\"", + char char1 = 'a', + char char2 = '\u200D', + double double1 = double.NegativeInfinity, + double double2 = double.PositiveInfinity, + double double3 = double.NaN, + double double4 = double.MaxValue, + double double5 = double.Epsilon, + float float1 = float.NegativeInfinity, + float float2 = float.PositiveInfinity, + float float3 = float.NaN, + float float4 = float.MinValue, + float float5 = float.Epsilon, + byte @byte = byte.MinValue, + decimal @decimal1 = decimal.MinValue, + decimal @decimal2 = decimal.MaxValue, + short @short = short.MinValue, + sbyte @sbyte = sbyte.MaxValue, + int @int = int.MinValue, + long @long = long.MaxValue, + ushort @ushort = ushort.MinValue, + uint @uint = uint.MaxValue, + ulong @ulong = ulong.MinValue) + { + Struct = @struct; + Enum1 = enum1; + Enum2 = enum2; + Enum3 = enum3; + Enum4 = enum4; + Enum5 = enum5; + Str1 = str1; + Str2 = str2; + Str3 = str3; + Char1 = char1; + Char2 = char2; + Double1 = double1; + Double2 = double2; + Double3 = double3; + Double4 = double4; + Float1 = float1; + Float2 = float2; + Float3 = float3; + Float4 = float4; + Byte = @byte; + Decimal1 = @decimal1; + Decimal2 = @decimal2; + Short = @short; + Sbyte = @sbyte; + Int = @int; + Long = @long; + UShort = @ushort; + UInt = @uint; + ULong = @ulong; + } + } } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/Serialization/ConstructorTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/Serialization/ConstructorTests.cs index deba91529ef37d..3f65e034f6768e 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/Serialization/ConstructorTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/Serialization/ConstructorTests.cs @@ -130,6 +130,7 @@ protected ConstructorTests_Metadata(JsonSerializerWrapperForString stringWrapper [JsonSerializable(typeof(LargeType_IgnoredProp_Bind_ParamWithDefaultValue))] [JsonSerializable(typeof(LargeType_IgnoredProp_Bind_Param))] [JsonSerializable(typeof(ClassWithIgnoredSameType))] + [JsonSerializable(typeof(ClassWithDefaultCtorParams))] internal sealed partial class ConstructorTestsContext_Metadata : JsonSerializerContext { } @@ -251,6 +252,7 @@ public ConstructorTests_Default() [JsonSerializable(typeof(LargeType_IgnoredProp_Bind_ParamWithDefaultValue))] [JsonSerializable(typeof(LargeType_IgnoredProp_Bind_Param))] [JsonSerializable(typeof(ClassWithIgnoredSameType))] + [JsonSerializable(typeof(ClassWithDefaultCtorParams))] internal sealed partial class ConstructorTestsContext_Default : JsonSerializerContext { } From 5af5538ae9e186ad72162dd724cd82022824202a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 7 Jan 2022 20:33:37 -0700 Subject: [PATCH 25/39] Suppress warnings about [Obsolete] member usage in JSON src-gen'd code (#63501) Co-authored-by: Layomi Akinrinade --- .../gen/JsonSourceGenerator.Emitter.cs | 5 ++- .../JsonSourceGeneratorTests.cs | 31 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs index 7cfda7f6a12be7..706f42be018bbf 100644 --- a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs +++ b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs @@ -149,7 +149,10 @@ private void AddSource(string fileName, string source, bool isRootContextDef = f bool isInGlobalNamespace = @namespace == JsonConstants.GlobalNamespaceValue; StringBuilder sb = new(@"// -#nullable enable"); +#nullable enable + +// Suppress warnings about [Obsolete] member usage in generated code. +#pragma warning disable CS0618"); if (!isInGlobalNamespace) { diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorTests.cs index 69ca5e498f5f96..cb297b56a0fa30 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorTests.cs @@ -521,5 +521,36 @@ public class MyType Assert.Equal(1, types.Count); Assert.Equal("HelloWorld.MyType", types.Keys.First()); } + + [Fact] + public static void NoWarningsDueToObsoleteMembers() + { + string source = @"using System; +using System.Text.Json.Serialization; + +namespace Test +{ + [JsonSerializable(typeof(ClassWithObsolete))] + public partial class JsonContext : JsonSerializerContext { } + + public class ClassWithObsolete + { + [Obsolete(""This is a test"")] + public bool Test { get; set; } + } +} +"; + + Compilation compilation = CompilationHelper.CreateCompilation(source); + JsonSourceGenerator generator = new JsonSourceGenerator(); + + Compilation newCompilation = CompilationHelper.RunGenerators(compilation, out _, generator); + ImmutableArray generatorDiags = newCompilation.GetDiagnostics(); + + // No diagnostics expected. + CompilationHelper.CheckDiagnosticMessages(DiagnosticSeverity.Info, generatorDiags, Array.Empty<(Location, string)>()); + CompilationHelper.CheckDiagnosticMessages(DiagnosticSeverity.Warning, generatorDiags, Array.Empty<(Location, string)>()); + CompilationHelper.CheckDiagnosticMessages(DiagnosticSeverity.Error, generatorDiags, Array.Empty<(Location, string)>()); + } } } From 26b0a12b3ca5eef7031898dea0351873ce89fb8b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 7 Jan 2022 20:35:01 -0700 Subject: [PATCH 26/39] [release/6.0] Fix TimeSpan support in sourcegen (#62191) * fix TimeSpan support in sourcegen * address feedback * Address feedback Co-authored-by: Eirik Tsarpalis Co-authored-by: Eric StJohn --- .../System.Text.Json/gen/JsonSourceGenerator.Parser.cs | 3 +++ .../RealWorldContextTests.cs | 4 +++- .../System.Text.Json.SourceGeneration.Tests/TestClasses.cs | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs index bc4b62669dd102..0204a12eed3341 100644 --- a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs +++ b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs @@ -80,6 +80,7 @@ private sealed class Parser private readonly Type _objectType; private readonly Type _stringType; + private readonly Type? _timeSpanType; private readonly Type? _dateTimeOffsetType; private readonly Type? _byteArrayType; private readonly Type? _guidType; @@ -202,6 +203,7 @@ public Parser(Compilation compilation, in JsonSourceGenerationContext sourceGene _booleanType = _metadataLoadContext.Resolve(SpecialType.System_Boolean); _charType = _metadataLoadContext.Resolve(SpecialType.System_Char); + _timeSpanType = _metadataLoadContext.Resolve(typeof(TimeSpan)); _dateTimeType = _metadataLoadContext.Resolve(SpecialType.System_DateTime); _nullableOfTType = _metadataLoadContext.Resolve(SpecialType.System_Nullable_T); _objectType = _metadataLoadContext.Resolve(SpecialType.System_Object); @@ -1509,6 +1511,7 @@ private void PopulateKnownTypes() _knownTypes.Add(_stringType); AddTypeIfNotNull(_knownTypes, _byteArrayType); + AddTypeIfNotNull(_knownTypes, _timeSpanType); AddTypeIfNotNull(_knownTypes, _dateTimeOffsetType); AddTypeIfNotNull(_knownTypes, _guidType); AddTypeIfNotNull(_knownTypes, _uriType); diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/RealWorldContextTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/RealWorldContextTests.cs index ee891562fffee7..ebfd93c01e3717 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/RealWorldContextTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/RealWorldContextTests.cs @@ -399,7 +399,8 @@ protected static ActiveOrUpcomingEvent CreateActiveOrUpcomingEvent() EndDate = DateTime.UtcNow.AddYears(1), Name = "Just a name", ImageUrl = "https://www.dotnetfoundation.org/theme/img/carousel/foundation-diagram-content.png", - StartDate = DateTime.UtcNow + StartDate = DateTime.UtcNow, + Offset = TimeSpan.FromHours(2) }; } @@ -413,6 +414,7 @@ protected static void VerifyActiveOrUpcomingEvent(ActiveOrUpcomingEvent expected Assert.Equal(expected.ImageUrl, obj.ImageUrl); Assert.Equal(expected.Name, obj.Name); Assert.Equal(expected.StartDate, obj.StartDate); + Assert.Equal(expected.Offset, obj.Offset); } protected static CampaignSummaryViewModel CreateCampaignSummaryViewModel() diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/TestClasses.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/TestClasses.cs index 887f65b80da83b..d28a73c7b24d55 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/TestClasses.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/TestClasses.cs @@ -60,6 +60,7 @@ public class ActiveOrUpcomingEvent public string Description { get; set; } public DateTimeOffset StartDate { get; set; } public DateTimeOffset EndDate { get; set; } + public TimeSpan Offset { get; set; } } public class CampaignSummaryViewModel From a9723f3e556af083b268c1b88ed86cbca906e5c9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 7 Jan 2022 20:35:54 -0700 Subject: [PATCH 27/39] Include properties on records for (de)serialization in source-gen (#63454) Co-authored-by: Layomi Akinrinade --- .../gen/Reflection/TypeWrapper.cs | 6 - .../CompilationHelper.cs | 53 +++++ .../JsonSourceGeneratorTests.cs | 202 +++++++++++++++++- 3 files changed, 251 insertions(+), 10 deletions(-) diff --git a/src/libraries/System.Text.Json/gen/Reflection/TypeWrapper.cs b/src/libraries/System.Text.Json/gen/Reflection/TypeWrapper.cs index 2d06d9ec50cd96..c084c6f614823c 100644 --- a/src/libraries/System.Text.Json/gen/Reflection/TypeWrapper.cs +++ b/src/libraries/System.Text.Json/gen/Reflection/TypeWrapper.cs @@ -396,12 +396,6 @@ public override PropertyInfo[] GetProperties(BindingFlags bindingAttr) { if (item is IPropertySymbol propertySymbol) { - // Skip auto-generated properties on records. - if (_typeSymbol.IsRecord && propertySymbol.DeclaringSyntaxReferences.Length == 0) - { - continue; - } - // Skip if: if ( // we want a static property and this is not static diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/CompilationHelper.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/CompilationHelper.cs index 8e3105e119238e..485a79cd333cd6 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/CompilationHelper.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/CompilationHelper.cs @@ -336,6 +336,59 @@ public partial class MyJsonContext : JsonSerializerContext return CreateCompilation(source); } + public static Compilation CreateReferencedLibRecordCompilation() + { + string source = @" + using System.Text.Json.Serialization; + + namespace ReferencedAssembly + { + public record LibRecord(int Id) + { + public string Address1 { get; set; } + public string Address2 { get; set; } + public string City { get; set; } + public string State { get; set; } + public string PostalCode { get; set; } + public string Name { get; set; } + [JsonInclude] + public string PhoneNumber; + [JsonInclude] + public string Country; + } + } +"; + + return CreateCompilation(source); + } + + public static Compilation CreateReferencedSimpleLibRecordCompilation() + { + string source = @" + using System.Text.Json.Serialization; + + namespace ReferencedAssembly + { + public record LibRecord + { + public int Id { get; set; } + public string Address1 { get; set; } + public string Address2 { get; set; } + public string City { get; set; } + public string State { get; set; } + public string PostalCode { get; set; } + public string Name { get; set; } + [JsonInclude] + public string PhoneNumber; + [JsonInclude] + public string Country; + } + } +"; + + return CreateCompilation(source); + } + internal static void CheckDiagnosticMessages( DiagnosticSeverity level, ImmutableArray diagnostics, diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorTests.cs index cb297b56a0fa30..b75bdf829a1966 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorTests.cs @@ -448,22 +448,216 @@ public void UsePrivates() CheckFieldsPropertiesMethods(myType, expectedFieldNames, expectedPropertyNames, expectedMethodNames); } + [Fact] + public void Record() + { + // Compile the referenced assembly first. + Compilation referencedCompilation = CompilationHelper.CreateReferencedLibRecordCompilation(); + + // Emit the image of the referenced assembly. + byte[] referencedImage = CompilationHelper.CreateAssemblyImage(referencedCompilation); + + string source = @" + using System.Text.Json.Serialization; + + namespace HelloWorld + { + [JsonSerializable(typeof(AppRecord))] + internal partial class JsonContext : JsonSerializerContext + { + } + + public record AppRecord(int Id) + { + public string Address1 { get; set; } + public string Address2 { get; set; } + public string City { get; set; } + public string State { get; set; } + public string PostalCode { get; set; } + public string Name { get; set; } + [JsonInclude] + public string PhoneNumber; + [JsonInclude] + public string Country; + } + }"; + + MetadataReference[] additionalReferences = { MetadataReference.CreateFromImage(referencedImage) }; + + Compilation compilation = CompilationHelper.CreateCompilation(source); + + JsonSourceGenerator generator = new JsonSourceGenerator(); + + Compilation newCompilation = CompilationHelper.RunGenerators(compilation, out ImmutableArray generatorDiags, generator); + + // Make sure compilation was successful. + CheckCompilationDiagnosticsErrors(generatorDiags); + CheckCompilationDiagnosticsErrors(newCompilation.GetDiagnostics()); + + Dictionary types = generator.GetSerializableTypes(); + + // Check base functionality of found types. + Assert.Equal(1, types.Count); + Type recordType = types["HelloWorld.AppRecord"]; + Assert.Equal("HelloWorld.AppRecord", recordType.FullName); + + // Check for received fields, properties and methods for NotMyType. + string[] expectedFieldsNames = { "Country", "PhoneNumber" }; + string[] expectedPropertyNames = { "Address1", "Address2", "City", "Id", "Name", "PostalCode", "State" }; + CheckFieldsPropertiesMethods(recordType, expectedFieldsNames, expectedPropertyNames); + + Assert.Equal(1, recordType.GetConstructors().Length); + } + + [Fact] + public void RecordInExternalAssembly() + { + // Compile the referenced assembly first. + Compilation referencedCompilation = CompilationHelper.CreateReferencedLibRecordCompilation(); + + // Emit the image of the referenced assembly. + byte[] referencedImage = CompilationHelper.CreateAssemblyImage(referencedCompilation); + + string source = @" + using System.Text.Json.Serialization; + using ReferencedAssembly; + + namespace HelloWorld + { + [JsonSerializable(typeof(LibRecord))] + internal partial class JsonContext : JsonSerializerContext + { + } + }"; + + MetadataReference[] additionalReferences = { MetadataReference.CreateFromImage(referencedImage) }; + + Compilation compilation = CompilationHelper.CreateCompilation(source, additionalReferences); + + JsonSourceGenerator generator = new JsonSourceGenerator(); + + Compilation newCompilation = CompilationHelper.RunGenerators(compilation, out ImmutableArray generatorDiags, generator); + + // Make sure compilation was successful. + CheckCompilationDiagnosticsErrors(generatorDiags); + CheckCompilationDiagnosticsErrors(newCompilation.GetDiagnostics()); + + Dictionary types = generator.GetSerializableTypes(); + + Assert.Equal(1, types.Count); + Type recordType = types["ReferencedAssembly.LibRecord"]; + Assert.Equal("ReferencedAssembly.LibRecord", recordType.FullName); + + string[] expectedFieldsNames = { "Country", "PhoneNumber" }; + string[] expectedPropertyNames = { "Address1", "Address2", "City", "Id", "Name", "PostalCode", "State" }; + CheckFieldsPropertiesMethods(recordType, expectedFieldsNames, expectedPropertyNames); + + Assert.Equal(1, recordType.GetConstructors().Length); + } + + [Fact] + public void RecordDerivedFromRecordInExternalAssembly() + { + // Compile the referenced assembly first. + Compilation referencedCompilation = CompilationHelper.CreateReferencedSimpleLibRecordCompilation(); + + // Emit the image of the referenced assembly. + byte[] referencedImage = CompilationHelper.CreateAssemblyImage(referencedCompilation); + + string source = @" + using System.Text.Json.Serialization; + using ReferencedAssembly; + + namespace HelloWorld + { + [JsonSerializable(typeof(AppRecord))] + internal partial class JsonContext : JsonSerializerContext + { + } + + internal record AppRecord : LibRecord + { + public string ExtraData { get; set; } + } + }"; + + MetadataReference[] additionalReferences = { MetadataReference.CreateFromImage(referencedImage) }; + + Compilation compilation = CompilationHelper.CreateCompilation(source, additionalReferences); + + JsonSourceGenerator generator = new JsonSourceGenerator(); + + Compilation newCompilation = CompilationHelper.RunGenerators(compilation, out ImmutableArray generatorDiags, generator); + + // Make sure compilation was successful. + CheckCompilationDiagnosticsErrors(generatorDiags); + CheckCompilationDiagnosticsErrors(newCompilation.GetDiagnostics()); + + Dictionary types = generator.GetSerializableTypes(); + + Assert.Equal(1, types.Count); + Type recordType = types["HelloWorld.AppRecord"]; + Assert.Equal("HelloWorld.AppRecord", recordType.FullName); + + string[] expectedFieldsNames = { "Country", "PhoneNumber" }; + string[] expectedPropertyNames = { "Address1", "Address2", "City", "ExtraData", "Id", "Name", "PostalCode", "State" }; + CheckFieldsPropertiesMethods(recordType, expectedFieldsNames, expectedPropertyNames, inspectBaseTypes: true); + + Assert.Equal(1, recordType.GetConstructors().Length); + } + private void CheckCompilationDiagnosticsErrors(ImmutableArray diagnostics) { Assert.Empty(diagnostics.Where(diagnostic => diagnostic.Severity == DiagnosticSeverity.Error)); } - private void CheckFieldsPropertiesMethods(Type type, string[] expectedFields, string[] expectedProperties, string[] expectedMethods) + private void CheckFieldsPropertiesMethods( + Type type, + string[] expectedFields, + string[] expectedProperties, + string[] expectedMethods = null, + bool inspectBaseTypes = false) { BindingFlags bindingFlags = BindingFlags.Public | BindingFlags.Instance; - string[] receivedFields = type.GetFields(bindingFlags).Select(field => field.Name).OrderBy(s => s).ToArray(); - string[] receivedProperties = type.GetProperties(bindingFlags).Select(property => property.Name).OrderBy(s => s).ToArray(); + string[] receivedFields; + string[] receivedProperties; + + if (!inspectBaseTypes) + { + receivedFields = type.GetFields(bindingFlags).Select(field => field.Name).OrderBy(s => s).ToArray(); + receivedProperties = type.GetProperties(bindingFlags).Select(property => property.Name).OrderBy(s => s).ToArray(); + } + else + { + List fields = new List(); + List props = new List(); + + Type currentType = type; + while (currentType != null) + { + fields.AddRange(currentType.GetFields(bindingFlags).Select(property => property.Name).OrderBy(s => s).ToArray()); + props.AddRange(currentType.GetProperties(bindingFlags).Select(property => property.Name).OrderBy(s => s).ToArray()); + currentType = currentType.BaseType; + } + + receivedFields = fields.ToArray(); + receivedProperties = props.ToArray(); + } + string[] receivedMethods = type.GetMethods().Select(method => method.Name).OrderBy(s => s).ToArray(); + Array.Sort(receivedFields); + Array.Sort(receivedProperties); + Array.Sort(receivedMethods); + Assert.Equal(expectedFields, receivedFields); Assert.Equal(expectedProperties, receivedProperties); - Assert.Equal(expectedMethods, receivedMethods); + + if (expectedMethods != null) + { + Assert.Equal(expectedMethods, receivedMethods); + } } // TODO: add test guarding against (de)serializing static classes. From 4d8bebca40a258682868e20988c45a3a753939fe Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 7 Jan 2022 22:36:51 -0500 Subject: [PATCH 28/39] [wasm] Backport build improvements, and fixes from #61581 (#62757) * [wasm] Disable native strip for tests * [wasm] improve fixup of symbol names * [wasm] Extract BuildProject params to BuildProjectOptions record * [wasm] Add WasmBuild.sln * [wasm] Add `@(NativeFileReference)` to up-to-date check items for VS VS does its own tracking of inputs/outputs too, and needs to be told about `@(NativeFileReference)` items, so a build can get triggered with F5 when a native file changes. Based on https://github.com/dotnet/project-system/blob/main/docs/up-to-date-check.md?rgh-link-date=2021-10-26T13%3A23%3A47Z#up-to-date-check Fixes https://github.com/dotnet/runtime/issues/60862 * MonoAOTCompiler: log failed output as error * [wasm] Improve error message when runtime pack cannot be found * [wasm] PInvokeTableGenerator: don't fail on errors in reading custom attributes Inspired by https://devdiv.visualstudio.com/DevDiv/_workitems/edit/1364890?src=WorkItemMention&src-action=artifact_link Works around a NRE in CustomAttributeData.AttributeType because it's ConstructorInfo is null. The mono side issue: https://github.com/mono/mono/issues/15340 * replace '.', '-', and '+' with '_' instead of their code --- eng/testing/tests.wasm.targets | 2 + src/mono/wasm/build/WasmApp.Native.targets | 4 + src/mono/wasm/build/WasmApp.targets | 2 +- src/mono/wasm/sln/WasmBuild.sln | 73 +++++++++++++++++++ src/mono/wasm/wasm.proj | 1 + src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 6 +- .../WasmAppBuilder/PInvokeTableGenerator.cs | 64 ++++++++++++++-- .../Wasm.Build.Tests/BuildPublishTests.cs | 57 ++++++++------- .../Wasm.Build.Tests/BuildTestBase.cs | 56 ++++++++------ .../InvariantGlobalizationTests.cs | 9 ++- .../Wasm.Build.Tests/LocalEMSDKTests.cs | 10 ++- .../Wasm.Build.Tests/MainWithArgsTests.cs | 7 +- .../Wasm.Build.Tests/NativeBuildTests.cs | 21 +++--- .../Wasm.Build.Tests/NativeLibraryTests.cs | 11 +-- .../NativeRebuildTestsBase.cs | 22 +++--- .../PInvokeTableGeneratorTests.cs | 17 +++-- .../Wasm.Build.Tests/RebuildTests.cs | 16 ++-- .../SatelliteAssembliesTests.cs | 45 ++++++------ .../Wasm.Build.Tests/WasmBuildAppTest.cs | 35 +++++---- .../WasmNativeDefaultsTests.cs | 12 +-- 20 files changed, 320 insertions(+), 150 deletions(-) create mode 100755 src/mono/wasm/sln/WasmBuild.sln diff --git a/eng/testing/tests.wasm.targets b/eng/testing/tests.wasm.targets index f68b4f9889a793..870135c5ed7589 100644 --- a/eng/testing/tests.wasm.targets +++ b/eng/testing/tests.wasm.targets @@ -8,6 +8,8 @@ true + + false diff --git a/src/mono/wasm/build/WasmApp.Native.targets b/src/mono/wasm/build/WasmApp.Native.targets index 22538f7ee777cf..7404e74554ac79 100644 --- a/src/mono/wasm/build/WasmApp.Native.targets +++ b/src/mono/wasm/build/WasmApp.Native.targets @@ -27,6 +27,10 @@ true + + + + <_MonoComponent Include="hot_reload;debugger" /> diff --git a/src/mono/wasm/build/WasmApp.targets b/src/mono/wasm/build/WasmApp.targets index 9c96d759a228a1..7642cf01694908 100644 --- a/src/mono/wasm/build/WasmApp.targets +++ b/src/mono/wasm/build/WasmApp.targets @@ -159,7 +159,7 @@ + Text="%24(MicrosoftNetCoreAppRuntimePackDir)='', and cannot find %25(ResolvedRuntimePack.PackageDirectory)=%(ResolvedRuntimePack.PackageDirectory). One of these need to be set to a valid path" /> diff --git a/src/mono/wasm/sln/WasmBuild.sln b/src/mono/wasm/sln/WasmBuild.sln new file mode 100755 index 00000000000000..10256d28e62648 --- /dev/null +++ b/src/mono/wasm/sln/WasmBuild.sln @@ -0,0 +1,73 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31722.452 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WasmBuildTasks", "..\..\..\tasks\WasmBuildTasks\WasmBuildTasks.csproj", "{D5BD9C0C-8A05-493E-BE45-13AF8286CD92}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WasmAppBuilder", "..\..\..\tasks\WasmAppBuilder\WasmAppBuilder.csproj", "{8DEBFDE2-B127-46D7-92CF-EEA6D1DA2554}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MonoAOTCompiler", "..\..\..\tasks\AotCompilerTask\MonoAOTCompiler.csproj", "{A9C02284-0387-42E7-BF78-47DF13656D5E}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wasm.Build.Tests", "..\..\..\tests\BuildWasmApps\Wasm.Build.Tests\Wasm.Build.Tests.csproj", "{94E18644-B0E5-4DBB-9CE4-EA1515ACE4C2}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DebuggerTestSuite", "..\debugger\DebuggerTestSuite\DebuggerTestSuite.csproj", "{4C0EE027-FC30-4167-B2CF-A6D18F00E08F}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BrowserDebugHost", "..\debugger\BrowserDebugHost\BrowserDebugHost.csproj", "{292A88FD-795F-467A-8801-B5B791CEF96E}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BrowserDebugProxy", "..\debugger\BrowserDebugProxy\BrowserDebugProxy.csproj", "{F5AE2AF5-3C30-45E3-A0C6-D962C51FE5E7}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WasmAppHost", "..\host\WasmAppHost.csproj", "{C7099764-EC2E-4FAF-9057-0321893DE4F8}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ApplyUpdateReferencedAssembly", "..\debugger\tests\ApplyUpdateReferencedAssembly\ApplyUpdateReferencedAssembly.csproj", "{75477B6F-DC8E-4002-88B8-017C992C568E}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {D5BD9C0C-8A05-493E-BE45-13AF8286CD92}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D5BD9C0C-8A05-493E-BE45-13AF8286CD92}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D5BD9C0C-8A05-493E-BE45-13AF8286CD92}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D5BD9C0C-8A05-493E-BE45-13AF8286CD92}.Release|Any CPU.Build.0 = Release|Any CPU + {8DEBFDE2-B127-46D7-92CF-EEA6D1DA2554}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8DEBFDE2-B127-46D7-92CF-EEA6D1DA2554}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8DEBFDE2-B127-46D7-92CF-EEA6D1DA2554}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8DEBFDE2-B127-46D7-92CF-EEA6D1DA2554}.Release|Any CPU.Build.0 = Release|Any CPU + {A9C02284-0387-42E7-BF78-47DF13656D5E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A9C02284-0387-42E7-BF78-47DF13656D5E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A9C02284-0387-42E7-BF78-47DF13656D5E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A9C02284-0387-42E7-BF78-47DF13656D5E}.Release|Any CPU.Build.0 = Release|Any CPU + {94E18644-B0E5-4DBB-9CE4-EA1515ACE4C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {94E18644-B0E5-4DBB-9CE4-EA1515ACE4C2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {94E18644-B0E5-4DBB-9CE4-EA1515ACE4C2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {94E18644-B0E5-4DBB-9CE4-EA1515ACE4C2}.Release|Any CPU.Build.0 = Release|Any CPU + {4C0EE027-FC30-4167-B2CF-A6D18F00E08F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4C0EE027-FC30-4167-B2CF-A6D18F00E08F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4C0EE027-FC30-4167-B2CF-A6D18F00E08F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4C0EE027-FC30-4167-B2CF-A6D18F00E08F}.Release|Any CPU.Build.0 = Release|Any CPU + {292A88FD-795F-467A-8801-B5B791CEF96E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {292A88FD-795F-467A-8801-B5B791CEF96E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {292A88FD-795F-467A-8801-B5B791CEF96E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {292A88FD-795F-467A-8801-B5B791CEF96E}.Release|Any CPU.Build.0 = Release|Any CPU + {F5AE2AF5-3C30-45E3-A0C6-D962C51FE5E7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F5AE2AF5-3C30-45E3-A0C6-D962C51FE5E7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F5AE2AF5-3C30-45E3-A0C6-D962C51FE5E7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F5AE2AF5-3C30-45E3-A0C6-D962C51FE5E7}.Release|Any CPU.Build.0 = Release|Any CPU + {75477B6F-DC8E-4002-88B8-017C992C568E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {75477B6F-DC8E-4002-88B8-017C992C568E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {75477B6F-DC8E-4002-88B8-017C992C568E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {75477B6F-DC8E-4002-88B8-017C992C568E}.Release|Any CPU.Build.0 = Release|Any CPU + {C7099764-EC2E-4FAF-9057-0321893DE4F8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C7099764-EC2E-4FAF-9057-0321893DE4F8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C7099764-EC2E-4FAF-9057-0321893DE4F8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C7099764-EC2E-4FAF-9057-0321893DE4F8}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {2BDE8FDE-4261-4B4D-8B54-ACC88B06C8D1} + EndGlobalSection +EndGlobal diff --git a/src/mono/wasm/wasm.proj b/src/mono/wasm/wasm.proj index 1051ff672f95b1..e7a38f7681c642 100644 --- a/src/mono/wasm/wasm.proj +++ b/src/mono/wasm/wasm.proj @@ -12,6 +12,7 @@ emcc $(ArtifactsObjDir)wasm <_EmccDefaultsRspPath>$(NativeBinDir)src\emcc-default.rsp + false diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index 87642d8d2fb32b..951015ed2ff913 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -744,13 +744,13 @@ private bool PrecompileLibrary(PrecompileArguments args) Log.LogMessage(importance, $"{msgPrefix}Exec (with response file contents expanded) in {args.WorkingDir}: {envStr}{CompilerBinaryPath} {File.ReadAllText(args.ResponseFilePath)}"); } - Log.LogMessage(importance, output); - if (exitCode != 0) { - Log.LogError($"Precompiling failed for {assembly}"); + Log.LogError($"Precompiling failed for {assembly}.{Environment.NewLine}{output}"); return false; } + + Log.LogMessage(importance, output); } catch (Exception ex) { diff --git a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs index 7b737715654f5f..c1c0d70a302db8 100644 --- a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs +++ b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs @@ -24,7 +24,7 @@ public class PInvokeTableGenerator : Task [Output] public string FileWrites { get; private set; } = string.Empty; - private static char[] s_charsToReplace = new[] { '.', '-', }; + private static char[] s_charsToReplace = new[] { '.', '-', '+' }; public override bool Execute() { @@ -88,7 +88,21 @@ public void GenPInvokeTable(string[] pinvokeModules, string[] assemblies) private void CollectPInvokes(List pinvokes, List callbacks, Type type) { - foreach (var method in type.GetMethods(BindingFlags.DeclaredOnly|BindingFlags.Public|BindingFlags.NonPublic|BindingFlags.Static|BindingFlags.Instance)) { + foreach (var method in type.GetMethods(BindingFlags.DeclaredOnly|BindingFlags.Public|BindingFlags.NonPublic|BindingFlags.Static|BindingFlags.Instance)) + { + try + { + CollectPInvokesForMethod(method); + } + catch (Exception ex) + { + Log.LogMessage(MessageImportance.Low, $"Could not get pinvoke, or callbacks for method {method.Name}: {ex}"); + continue; + } + } + + void CollectPInvokesForMethod(MethodInfo method) + { if ((method.Attributes & MethodAttributes.PinvokeImpl) != 0) { var dllimport = method.CustomAttributes.First(attr => attr.AttributeType.Name == "DllImportAttribute"); @@ -164,7 +178,8 @@ private void EmitPInvokeTable(StreamWriter w, Dictionary modules Where(l => l.Module == module && !l.Skip). OrderBy(l => l.EntryPoint). GroupBy(d => d.EntryPoint). - Select (l => "{\"" + l.Key + "\", " + l.Key + "}, // " + string.Join (", ", l.Select(c => c.Method.DeclaringType!.Module!.Assembly!.GetName ()!.Name!).Distinct().OrderBy(n => n))); + Select (l => "{\"" + FixupSymbolName(l.Key) + "\", " + FixupSymbolName(l.Key) + "}, " + + "// " + string.Join (", ", l.Select(c => c.Method.DeclaringType!.Module!.Assembly!.GetName ()!.Name!).Distinct().OrderBy(n => n))); foreach (var pinvoke in assemblies_pinvokes) { w.WriteLine (pinvoke); @@ -216,6 +231,45 @@ static bool ShouldTreatAsVariadic(PInvoke[] candidates) } } + private static string FixupSymbolName(string name) + { + UTF8Encoding utf8 = new(); + byte[] bytes = utf8.GetBytes(name); + StringBuilder sb = new(); + + foreach (byte b in bytes) + { + if ((b >= (byte)'0' && b <= (byte)'9') || + (b >= (byte)'a' && b <= (byte)'z') || + (b >= (byte)'A' && b <= (byte)'Z') || + (b == (byte)'_')) + { + sb.Append((char) b); + } + else if (s_charsToReplace.Contains((char) b)) + { + sb.Append('_'); + } + else + { + sb.Append($"_{b:X}_"); + } + } + + return sb.ToString(); + } + + private static string SymbolNameForMethod(MethodInfo method) + { + StringBuilder sb = new(); + Type? type = method.DeclaringType; + sb.Append($"{type!.Module!.Assembly!.GetName()!.Name!}_"); + sb.Append($"{(type!.IsNested ? type!.FullName : type!.Name)}_"); + sb.Append(method.Name); + + return FixupSymbolName(sb.ToString()); + } + private string MapType (Type t) { string name = t.Name; @@ -262,7 +316,7 @@ private static bool TryIsMethodGetParametersUnsupported(MethodInfo method, [NotN if (method.Name == "EnumCalendarInfo") { // FIXME: System.Reflection.MetadataLoadContext can't decode function pointer types // https://github.com/dotnet/runtime/issues/43791 - sb.Append($"int {pinvoke.EntryPoint} (int, int, int, int, int);"); + sb.Append($"int {FixupSymbolName(pinvoke.EntryPoint)} (int, int, int, int, int);"); return sb.ToString(); } @@ -274,7 +328,7 @@ private static bool TryIsMethodGetParametersUnsupported(MethodInfo method, [NotN } sb.Append(MapType(method.ReturnType)); - sb.Append($" {pinvoke.EntryPoint} ("); + sb.Append($" {FixupSymbolName(pinvoke.EntryPoint)} ("); int pindex = 0; var pars = method.GetParameters(); foreach (var p in pars) { diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildPublishTests.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildPublishTests.cs index 42eb96633f9b2a..1e25eb02a3f863 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildPublishTests.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildPublishTests.cs @@ -33,11 +33,14 @@ public void BuildThenPublishNoAOT(BuildArgs buildArgs, RunHost host, string id) // no relinking for build bool relinked = false; BuildProject(buildArgs, - initProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), - dotnetWasmFromRuntimePack: !relinked, id: id, - createProject: true, - publish: false); + new BuildProjectOptions( + InitProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), + DotnetWasmFromRuntimePack: !relinked, + CreateProject: true, + Publish: false + )); + Run(); @@ -53,10 +56,11 @@ public void BuildThenPublishNoAOT(BuildArgs buildArgs, RunHost host, string id) relinked = buildArgs.Config == "Release"; BuildProject(buildArgs, id: id, - dotnetWasmFromRuntimePack: !relinked, - createProject: false, - publish: true, - useCache: false); + new BuildProjectOptions( + DotnetWasmFromRuntimePack: !relinked, + CreateProject: false, + Publish: true, + UseCache: false)); Run(); @@ -79,12 +83,13 @@ public void BuildThenPublishWithAOT(BuildArgs buildArgs, RunHost host, string id // no relinking for build bool relinked = false; (_, string output) = BuildProject(buildArgs, - initProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), - dotnetWasmFromRuntimePack: !relinked, - id: id, - createProject: true, - publish: false, - label: "first_build"); + id, + new BuildProjectOptions( + InitProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), + DotnetWasmFromRuntimePack: !relinked, + CreateProject: true, + Publish: false, + Label: "first_build")); BuildPaths paths = GetBuildPaths(buildArgs); var pathsDict = GetFilesTable(buildArgs, paths, unchanged: false); @@ -109,11 +114,12 @@ public void BuildThenPublishWithAOT(BuildArgs buildArgs, RunHost host, string id // relink by default for Release+publish (_, output) = BuildProject(buildArgs, id: id, - dotnetWasmFromRuntimePack: false, - createProject: false, - publish: true, - useCache: false, - label: "first_publish"); + new BuildProjectOptions( + DotnetWasmFromRuntimePack: false, + CreateProject: false, + Publish: true, + UseCache: false, + Label: "first_publish")); var publishStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); Assert.True(publishStat["pinvoke.o"].Exists); @@ -125,12 +131,13 @@ public void BuildThenPublishWithAOT(BuildArgs buildArgs, RunHost host, string id // second build (_, output) = BuildProject(buildArgs, - initProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), - dotnetWasmFromRuntimePack: !relinked, - id: id, - createProject: true, - publish: false, - label: "second_build"); + id: id, + new BuildProjectOptions( + InitProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), + DotnetWasmFromRuntimePack: !relinked, + CreateProject: true, + Publish: false, + Label: "second_build")); var secondBuildStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); // no relinking, or AOT diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildTestBase.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildTestBase.cs index b6fab5c6c434b7..3472abb9b24256 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildTestBase.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/BuildTestBase.cs @@ -280,18 +280,10 @@ protected static BuildArgs ExpandBuildArgs(BuildArgs buildArgs, string extraProp public (string projectDir, string buildOutput) BuildProject(BuildArgs buildArgs, string id, - Action? initProject = null, - bool? dotnetWasmFromRuntimePack = null, - bool hasIcudt = true, - bool useCache = true, - bool expectSuccess = true, - bool createProject = true, - bool publish = true, - string? verbosity=null, - string? label=null) - { - string msgPrefix = label != null ? $"[{label}] " : string.Empty; - if (useCache && _buildContext.TryGetBuildFor(buildArgs, out BuildProduct? product)) + BuildProjectOptions options) + { + string msgPrefix = options.Label != null ? $"[{options.Label}] " : string.Empty; + if (options.UseCache && _buildContext.TryGetBuildFor(buildArgs, out BuildProduct? product)) { Console.WriteLine ($"Using existing build found at {product.ProjectDir}, with build log at {product.LogFile}"); @@ -303,33 +295,35 @@ protected static BuildArgs ExpandBuildArgs(BuildArgs buildArgs, string extraProp return (_projectDir, "FIXME"); } - if (createProject) + if (options.CreateProject) { InitPaths(id); InitProjectDir(_projectDir); - initProject?.Invoke(); + options.InitProject?.Invoke(); File.WriteAllText(Path.Combine(_projectDir, $"{buildArgs.ProjectName}.csproj"), buildArgs.ProjectFileContents); File.Copy(Path.Combine(AppContext.BaseDirectory, "runtime-test.js"), Path.Combine(_projectDir, "runtime-test.js")); } else if (_projectDir is null) { - throw new Exception("_projectDir should be set, to use createProject=false"); + throw new Exception("_projectDir should be set, to use options.createProject=false"); } StringBuilder sb = new(); - sb.Append(publish ? "publish" : "build"); + sb.Append(options.Publish ? "publish" : "build"); + if (options.Publish && options.BuildOnlyAfterPublish) + sb.Append(" -p:WasmBuildOnlyAfterPublish=true"); sb.Append($" {s_buildEnv.DefaultBuildArgs}"); sb.Append($" /p:Configuration={buildArgs.Config}"); - string logFileSuffix = label == null ? string.Empty : label.Replace(' ', '_'); + string logFileSuffix = options.Label == null ? string.Empty : options.Label.Replace(' ', '_'); string logFilePath = Path.Combine(_logPath, $"{buildArgs.ProjectName}{logFileSuffix}.binlog"); _testOutput.WriteLine($"-------- Building ---------"); _testOutput.WriteLine($"Binlog path: {logFilePath}"); Console.WriteLine($"Binlog path: {logFilePath}"); sb.Append($" /bl:\"{logFilePath}\" /nologo"); - sb.Append($" /fl /flp:\"v:diag,LogFile={logFilePath}.log\" /v:{verbosity ?? "minimal"}"); + sb.Append($" /fl /flp:\"v:diag,LogFile={logFilePath}.log\" /v:{options.Verbosity ?? "minimal"}"); if (buildArgs.ExtraBuildArgs != null) sb.Append($" {buildArgs.ExtraBuildArgs} "); @@ -338,26 +332,26 @@ protected static BuildArgs ExpandBuildArgs(BuildArgs buildArgs, string extraProp (int exitCode, string buildOutput) result; try { - result = AssertBuild(sb.ToString(), id, expectSuccess: expectSuccess, envVars: s_buildEnv.EnvVars); + result = AssertBuild(sb.ToString(), id, expectSuccess: options.ExpectSuccess, envVars: s_buildEnv.EnvVars); //AssertRuntimePackPath(result.buildOutput); // check that we are using the correct runtime pack! - if (expectSuccess) + if (options.ExpectSuccess) { string bundleDir = Path.Combine(GetBinDir(config: buildArgs.Config), "AppBundle"); - AssertBasicAppBundle(bundleDir, buildArgs.ProjectName, buildArgs.Config, hasIcudt, dotnetWasmFromRuntimePack ?? !buildArgs.AOT); + AssertBasicAppBundle(bundleDir, buildArgs.ProjectName, buildArgs.Config, options.HasIcudt, options.DotnetWasmFromRuntimePack ?? !buildArgs.AOT); } - if (useCache) + if (options.UseCache) _buildContext.CacheBuild(buildArgs, new BuildProduct(_projectDir, logFilePath, true)); return (_projectDir, result.buildOutput); } catch { - if (useCache) + if (options.UseCache) _buildContext.CacheBuild(buildArgs, new BuildProduct(_projectDir, logFilePath, false)); throw; } @@ -860,4 +854,18 @@ public record BuildArgs(string ProjectName, public record BuildProduct(string ProjectDir, string LogFile, bool Result); internal record FileStat (bool Exists, DateTime LastWriteTimeUtc, long Length, string FullPath); internal record BuildPaths(string ObjWasmDir, string ObjDir, string BinDir, string BundleDir); - } + + public record BuildProjectOptions + ( + Action? InitProject = null, + bool? DotnetWasmFromRuntimePack = null, + bool HasIcudt = true, + bool UseCache = true, + bool ExpectSuccess = true, + bool CreateProject = true, + bool Publish = true, + bool BuildOnlyAfterPublish = true, + string? Verbosity = null, + string? Label = null + ); +} diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/InvariantGlobalizationTests.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/InvariantGlobalizationTests.cs index 7d66bc32a49a11..f34cb9cde7d655 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/InvariantGlobalizationTests.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/InvariantGlobalizationTests.cs @@ -74,10 +74,11 @@ private void TestInvariantGlobalization(BuildArgs buildArgs, bool? invariantGlob "; BuildProject(buildArgs, - initProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText), - id: id, - dotnetWasmFromRuntimePack: dotnetWasmFromRuntimePack, - hasIcudt: invariantGlobalization == null || invariantGlobalization.Value == false); + id: id, + new BuildProjectOptions( + InitProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText), + DotnetWasmFromRuntimePack: dotnetWasmFromRuntimePack, + HasIcudt: invariantGlobalization == null || invariantGlobalization.Value == false)); if (invariantGlobalization == true) { diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/LocalEMSDKTests.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/LocalEMSDKTests.cs index 30ed330bf17910..5a85ae935bb916 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/LocalEMSDKTests.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/LocalEMSDKTests.cs @@ -30,9 +30,10 @@ public void AOT_ErrorWhenMissingEMSDK(BuildArgs buildArgs, string emsdkPath, str buildArgs = ExpandBuildArgs(buildArgs); (_, string buildOutput) = BuildProject(buildArgs, - initProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), id: id, - expectSuccess: false); + new BuildProjectOptions( + InitProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), + ExpectSuccess: false)); Assert.Matches(errorPattern, buildOutput); } @@ -52,9 +53,10 @@ public void Relinking_ErrorWhenMissingEMSDK(BuildArgs buildArgs, string emsdkPat buildArgs = ExpandBuildArgs(buildArgs, extraProperties: "true"); (_, string buildOutput) = BuildProject(buildArgs, - initProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), id: id, - expectSuccess: false); + new BuildProjectOptions( + InitProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), + ExpectSuccess: false)); Assert.Matches(errorPattern, buildOutput); } diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/MainWithArgsTests.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/MainWithArgsTests.cs index b97a0490d2e872..9155d2a97f2ce5 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/MainWithArgsTests.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/MainWithArgsTests.cs @@ -84,9 +84,10 @@ void TestMainWithArgs(string projectNamePrefix, Console.WriteLine ($"-- args: {buildArgs}, name: {projectName}"); BuildProject(buildArgs, - initProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText), - id: id, - dotnetWasmFromRuntimePack: dotnetWasmFromRuntimePack); + id: id, + new BuildProjectOptions( + InitProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText), + DotnetWasmFromRuntimePack: dotnetWasmFromRuntimePack)); RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42 + args.Length, args: string.Join(' ', args), test: output => diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeBuildTests.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeBuildTests.cs index 3453f9922dbbad..304161ef086d1f 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeBuildTests.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeBuildTests.cs @@ -32,9 +32,10 @@ public void SimpleNativeBuild(BuildArgs buildArgs, RunHost host, string id) buildArgs = ExpandBuildArgs(buildArgs, extraProperties: "true"); BuildProject(buildArgs, - initProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), - dotnetWasmFromRuntimePack: false, - id: id); + id: id, + new BuildProjectOptions( + InitProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), + DotnetWasmFromRuntimePack: false)); RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, test: output => {}, @@ -57,10 +58,11 @@ public void MonoAOTCross_WorksWithNoTrimming(BuildArgs buildArgs, string id) (_, string output) = BuildProject( buildArgs, - initProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), - dotnetWasmFromRuntimePack: false, id: id, - expectSuccess: false); + new BuildProjectOptions( + InitProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), + DotnetWasmFromRuntimePack: false, + ExpectSuccess: false)); Assert.Contains("Stopping after AOT", output); } @@ -89,9 +91,10 @@ public void IntermediateBitcodeToObjectFilesAreNotLLVMIR(BuildArgs buildArgs, st buildArgs = ExpandBuildArgs(buildArgs, insertAtEnd: printFileTypeTarget); (_, string output) = BuildProject(buildArgs, - initProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), - dotnetWasmFromRuntimePack: false, - id: id); + id: id, + new BuildProjectOptions( + InitProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), + DotnetWasmFromRuntimePack: false)); if (!output.Contains("** wasm-dis exit code: 0")) throw new XunitException($"Expected to successfully run wasm-dis on System.Private.CoreLib.dll.o ." diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeLibraryTests.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeLibraryTests.cs index eb306286ba5e0d..e8892a9575d017 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeLibraryTests.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeLibraryTests.cs @@ -37,8 +37,8 @@ public void ProjectWithNativeReference(BuildArgs buildArgs, RunHost host, string } BuildProject(buildArgs, - dotnetWasmFromRuntimePack: false, - id: id); + id: id, + new BuildProjectOptions(DotnetWasmFromRuntimePack: false)); string output = RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 0, test: output => {}, @@ -81,9 +81,10 @@ public static int Main() }"; BuildProject(buildArgs, - initProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText), - dotnetWasmFromRuntimePack: false, - id: id); + id: id, + new BuildProjectOptions( + InitProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText), + DotnetWasmFromRuntimePack: false)); string output = RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 0, test: output => {}, diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NativeRebuildTestsBase.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NativeRebuildTestsBase.cs index cd0d3ac4396349..da32a9f9e2ea1b 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NativeRebuildTestsBase.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NativeRebuildTestsBase.cs @@ -48,11 +48,12 @@ public NativeRebuildTestsBase(ITestOutputHelper output, SharedBuildPerTestClassF { buildArgs = GenerateProjectContents(buildArgs, nativeRelink, invariant, extraProperties); BuildProject(buildArgs, - initProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText), - dotnetWasmFromRuntimePack: false, - hasIcudt: !invariant, - id: id, - createProject: true); + id: id, + new BuildProjectOptions( + InitProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText), + DotnetWasmFromRuntimePack: false, + HasIcudt: !invariant, + CreateProject: true)); RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: RunHost.V8, id: id); return (buildArgs, GetBuildPaths(buildArgs)); @@ -80,11 +81,12 @@ protected string Rebuild(bool nativeRelink, bool invariant, BuildArgs buildArgs, Console.WriteLine($"{Environment.NewLine}Rebuilding with no changes ..{Environment.NewLine}"); (_, string output) = BuildProject(buildArgs, id: id, - dotnetWasmFromRuntimePack: false, - hasIcudt: !invariant, - createProject: false, - useCache: false, - verbosity: verbosity); + new BuildProjectOptions( + DotnetWasmFromRuntimePack: false, + HasIcudt: !invariant, + CreateProject: false, + UseCache: false, + Verbosity: verbosity)); return output; } diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs index 27783ae85c95ad..feada6bed9b99c 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs @@ -123,15 +123,16 @@ public static int Main() extraProperties: "true<_WasmDevel>true"); (_, string output) = BuildProject(buildArgs, - initProject: () => - { - File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText); - File.Copy(Path.Combine(BuildEnvironment.TestAssetsPath, "native-libs", filename), - Path.Combine(_projectDir!, filename)); - }, - publish: buildArgs.AOT, id: id, - dotnetWasmFromRuntimePack: false); + new BuildProjectOptions( + InitProject: () => + { + File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText); + File.Copy(Path.Combine(BuildEnvironment.TestAssetsPath, "native-libs", filename), + Path.Combine(_projectDir!, filename)); + }, + Publish: buildArgs.AOT, + DotnetWasmFromRuntimePack: false)); return (buildArgs, output); } diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/RebuildTests.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/RebuildTests.cs index 961c141348275d..ba65b2445aabf9 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/RebuildTests.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/RebuildTests.cs @@ -35,10 +35,11 @@ public void NoOpRebuild(BuildArgs buildArgs, RunHost host, string id) buildArgs = ExpandBuildArgs(buildArgs); BuildProject(buildArgs, - initProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), - dotnetWasmFromRuntimePack: true, - id: id, - createProject: true); + id: id, + new BuildProjectOptions( + InitProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), + DotnetWasmFromRuntimePack: true, + CreateProject: true)); Run(); @@ -52,9 +53,10 @@ public void NoOpRebuild(BuildArgs buildArgs, RunHost host, string id) // no-op Rebuild BuildProject(buildArgs, id: id, - dotnetWasmFromRuntimePack: true, - createProject: false, - useCache: false); + new BuildProjectOptions( + DotnetWasmFromRuntimePack: true, + CreateProject: false, + UseCache: false)); Run(); diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/SatelliteAssembliesTests.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/SatelliteAssembliesTests.cs index c3a40b470200a6..829f282df105ea 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/SatelliteAssembliesTests.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/SatelliteAssembliesTests.cs @@ -49,13 +49,14 @@ public void ResourcesFromMainAssembly(BuildArgs buildArgs, extraProperties: nativeRelink ? $"true" : string.Empty); BuildProject(buildArgs, - initProject: () => - { - Utils.DirectoryCopy(Path.Combine(BuildEnvironment.TestAssetsPath, "resx"), Path.Combine(_projectDir!, "resx")); - CreateProgramForCultureTest(_projectDir!, $"{projectName}.resx.words", "TestClass"); - }, - dotnetWasmFromRuntimePack: dotnetWasmFromRuntimePack, - id: id); + id: id, + new BuildProjectOptions( + InitProject: () => + { + Utils.DirectoryCopy(Path.Combine(BuildEnvironment.TestAssetsPath, "resx"), Path.Combine(_projectDir!, "resx")); + CreateProgramForCultureTest(_projectDir!, $"{projectName}.resx.words", "TestClass"); + }, + DotnetWasmFromRuntimePack: dotnetWasmFromRuntimePack)); string output = RunAndTestWasmApp( buildArgs, expectedExitCode: 42, @@ -86,17 +87,18 @@ public void ResourcesFromProjectReference(BuildArgs buildArgs, extraItems: $""); BuildProject(buildArgs, - dotnetWasmFromRuntimePack: dotnetWasmFromRuntimePack, - id: id, - initProject: () => - { - string rootDir = _projectDir!; - _projectDir = Path.Combine(rootDir, projectName); - - Directory.CreateDirectory(_projectDir); - Utils.DirectoryCopy(Path.Combine(BuildEnvironment.TestAssetsPath, "SatelliteAssemblyFromProjectRef"), rootDir); - CreateProgramForCultureTest(_projectDir, "LibraryWithResources.resx.words", "LibraryWithResources.Class1"); - }); + id: id, + new BuildProjectOptions( + DotnetWasmFromRuntimePack: dotnetWasmFromRuntimePack, + InitProject: () => + { + string rootDir = _projectDir!; + _projectDir = Path.Combine(rootDir, projectName); + + Directory.CreateDirectory(_projectDir); + Utils.DirectoryCopy(Path.Combine(BuildEnvironment.TestAssetsPath, "SatelliteAssemblyFromProjectRef"), rootDir); + CreateProgramForCultureTest(_projectDir, "LibraryWithResources.resx.words", "LibraryWithResources.Class1"); + })); string output = RunAndTestWasmApp(buildArgs, expectedExitCode: 42, @@ -121,9 +123,10 @@ public void CheckThatSatelliteAssembliesAreNotAOTed(BuildArgs buildArgs, string extraItems: $""); BuildProject(buildArgs, - initProject: () => CreateProgramForCultureTest(_projectDir!, $"{projectName}.words", "TestClass"), - dotnetWasmFromRuntimePack: false, - id: id); + id: id, + new BuildProjectOptions( + InitProject: () => CreateProgramForCultureTest(_projectDir!, $"{projectName}.words", "TestClass"), + DotnetWasmFromRuntimePack: false)); var bitCodeFileNames = Directory.GetFileSystemEntries(Path.Combine(_projectDir!, "obj"), "*.dll.bc", SearchOption.AllDirectories) .Select(path => Path.GetFileName(path)) diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/WasmBuildAppTest.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/WasmBuildAppTest.cs index 412bf4a3da2eae..d5df20a90ff547 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/WasmBuildAppTest.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/WasmBuildAppTest.cs @@ -110,13 +110,14 @@ public void PropertiesFromRuntimeConfigJson(BuildArgs buildArgs, RunHost host, s }"; BuildProject(buildArgs, - initProject: () => - { - File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText); - File.WriteAllText(Path.Combine(_projectDir!, "runtimeconfig.template.json"), runtimeConfigTemplateJson); - }, - id: id, - dotnetWasmFromRuntimePack: !(buildArgs.AOT || buildArgs.Config == "Release")); + id: id, + new BuildProjectOptions( + InitProject: () => + { + File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText); + File.WriteAllText(Path.Combine(_projectDir!, "runtimeconfig.template.json"), runtimeConfigTemplateJson); + }, + DotnetWasmFromRuntimePack: !(buildArgs.AOT || buildArgs.Config == "Release"))); RunAndTestWasmApp(buildArgs, expectedExitCode: 42, test: output => Assert.Contains("test_runtimeconfig_json: 25", output), host: host, id: id); @@ -139,12 +140,13 @@ public void PropertiesFromCsproj(BuildArgs buildArgs, RunHost host, string id) "; BuildProject(buildArgs, - initProject: () => - { - File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText); - }, - id: id, - dotnetWasmFromRuntimePack: !(buildArgs.AOT || buildArgs.Config == "Release")); + id: id, + new BuildProjectOptions( + InitProject: () => + { + File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText); + }, + DotnetWasmFromRuntimePack: !(buildArgs.AOT || buildArgs.Config == "Release"))); RunAndTestWasmApp(buildArgs, expectedExitCode: 42, test: output => Assert.Contains("System.Threading.ThreadPool.MaxThreads: 20", output), host: host, id: id); @@ -165,9 +167,10 @@ void TestMain(string projectName, dotnetWasmFromRuntimePack = !(buildArgs.AOT || buildArgs.Config == "Release"); BuildProject(buildArgs, - initProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText), - id: id, - dotnetWasmFromRuntimePack: dotnetWasmFromRuntimePack); + id: id, + new BuildProjectOptions( + InitProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText), + DotnetWasmFromRuntimePack: dotnetWasmFromRuntimePack)); RunAndTestWasmApp(buildArgs, expectedExitCode: 42, test: output => Assert.Contains("Hello, World!", output), host: host, id: id); diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/WasmNativeDefaultsTests.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/WasmNativeDefaultsTests.cs index 64d008549e8ec1..00674bf4630dcc 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/WasmNativeDefaultsTests.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/WasmNativeDefaultsTests.cs @@ -89,11 +89,13 @@ private string CheckWasmNativeDefaultValue(string projectName, insertAtEnd: printValueTarget); (_, string output) = BuildProject(buildArgs, - initProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), - dotnetWasmFromRuntimePack: dotnetWasmFromRuntimePack, - id: Path.GetRandomFileName(), - expectSuccess: false, - useCache: false); + id: Path.GetRandomFileName(), + new BuildProjectOptions( + InitProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), + DotnetWasmFromRuntimePack: dotnetWasmFromRuntimePack, + ExpectSuccess: false, + UseCache: false, + BuildOnlyAfterPublish: false)); return output; } From dfcba66b423c57b8ae2a000f144e3b8ea09ca186 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 7 Jan 2022 20:38:30 -0700 Subject: [PATCH 29/39] disable tests unstable on Windows 11 (#63527) Co-authored-by: wfurt --- .../tests/FunctionalTests/LoggingTest.cs | 7 +++++++ .../FunctionalTests/SslStreamNetworkStreamTest.cs | 6 ++++++ .../FunctionalTests/SslStreamStreamToStreamTest.cs | 10 +++++++++- .../FunctionalTests/SslStreamSystemDefaultsTest.cs | 8 +++++++- 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.Net.Security/tests/FunctionalTests/LoggingTest.cs b/src/libraries/System.Net.Security/tests/FunctionalTests/LoggingTest.cs index 60320d78a0ce2e..1b99f0cd4d85ae 100644 --- a/src/libraries/System.Net.Security/tests/FunctionalTests/LoggingTest.cs +++ b/src/libraries/System.Net.Security/tests/FunctionalTests/LoggingTest.cs @@ -5,6 +5,7 @@ using System.Diagnostics; using System.Diagnostics.Tracing; using Microsoft.DotNet.RemoteExecutor; +using Microsoft.DotNet.XUnitExtensions; using Xunit; namespace System.Net.Security.Tests @@ -26,6 +27,12 @@ public void EventSource_ExistsWithCorrectId() [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] public void EventSource_EventsRaisedAsExpected() { + if (PlatformDetection.IsWindows10Version22000OrGreater) + { + // [ActiveIssue("https://github.com/dotnet/runtime/issues/58927")] + throw new SkipTestException("Unstable on Windows 11"); + } + RemoteExecutor.Invoke(() => { using (var listener = new TestEventListener("Private.InternalDiagnostics.System.Net.Security", EventLevel.Verbose)) diff --git a/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamNetworkStreamTest.cs b/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamNetworkStreamTest.cs index 3428ba78845d48..285b6f1ff87952 100644 --- a/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamNetworkStreamTest.cs +++ b/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamNetworkStreamTest.cs @@ -413,6 +413,12 @@ await Assert.ThrowsAsync(()=> [PlatformSpecific(TestPlatforms.Windows)] public async Task SslStream_NegotiateClientCertificateAsyncTls13_Succeeds(bool sendClientCertificate) { + if (PlatformDetection.IsWindows10Version22000OrGreater) + { + // [ActiveIssue("https://github.com/dotnet/runtime/issues/58927")] + throw new SkipTestException("Unstable on Windows 11"); + } + bool negotiateClientCertificateCalled = false; using CancellationTokenSource cts = new CancellationTokenSource(); cts.CancelAfter(TestConfiguration.PassingTestTimeout); diff --git a/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamStreamToStreamTest.cs b/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamStreamToStreamTest.cs index 3ecfbc4e92493f..b05fbc6f53bc44 100644 --- a/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamStreamToStreamTest.cs +++ b/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamStreamToStreamTest.cs @@ -11,6 +11,7 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using Microsoft.DotNet.XUnitExtensions; using Xunit; namespace System.Net.Security.Tests @@ -66,11 +67,18 @@ public static IEnumerable SslStream_StreamToStream_Authentication_Succ } } - [Theory] + [ConditionalTheory] [MemberData(nameof(SslStream_StreamToStream_Authentication_Success_MemberData))] [SkipOnPlatform(TestPlatforms.iOS | TestPlatforms.tvOS, "X509 certificate store is not supported on iOS or tvOS.")] public async Task SslStream_StreamToStream_Authentication_Success(X509Certificate serverCert = null, X509Certificate clientCert = null) { + + if (PlatformDetection.IsWindows10Version22000OrGreater) + { + // [ActiveIssue("https://github.com/dotnet/runtime/issues/58927")] + throw new SkipTestException("Unstable on Windows 11"); + } + (Stream stream1, Stream stream2) = TestHelper.GetConnectedStreams(); using (var client = new SslStream(stream1, false, AllowAnyServerCertificate)) using (var server = new SslStream(stream2, false, delegate { return true; })) diff --git a/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamSystemDefaultsTest.cs b/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamSystemDefaultsTest.cs index 5380b4027a6f76..8e6b2176f62cbc 100644 --- a/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamSystemDefaultsTest.cs +++ b/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamSystemDefaultsTest.cs @@ -8,7 +8,7 @@ using System.Security.Cryptography.X509Certificates; using System.Security.Authentication; using System.Threading.Tasks; - +using Microsoft.DotNet.XUnitExtensions; using Xunit; namespace System.Net.Security.Tests @@ -76,6 +76,12 @@ public static IEnumerable OneOrBothUseDefaulData() [MemberData(nameof(OneOrBothUseDefaulData))] public async Task ClientAndServer_OneOrBothUseDefault_Ok(SslProtocols? clientProtocols, SslProtocols? serverProtocols) { + if (PlatformDetection.IsWindows10Version22000OrGreater) + { + // [ActiveIssue("https://github.com/dotnet/runtime/issues/58927")] + throw new SkipTestException("Unstable on Windows 11"); + } + using (X509Certificate2 serverCertificate = Configuration.Certificates.GetServerCertificate()) using (X509Certificate2 clientCertificate = Configuration.Certificates.GetClientCertificate()) { From a16c3221cd06a6a2feb1b8d586f0a6bc4a2123f7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 8 Jan 2022 12:14:51 -0800 Subject: [PATCH 30/39] [release/6.0] Suppress SendPacketsElement_FileStreamMultiPartMixed_MultipleFileStreams_Success test on Windows 11 (#63526) --- .../tests/FunctionalTests/SendPacketsAsync.cs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendPacketsAsync.cs b/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendPacketsAsync.cs index 29c24352c1a76c..65764b452f9908 100644 --- a/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendPacketsAsync.cs +++ b/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendPacketsAsync.cs @@ -6,6 +6,8 @@ using System.Net.Test.Common; using System.Threading; +using Microsoft.DotNet.XUnitExtensions; + using Xunit; using Xunit.Abstractions; @@ -525,7 +527,10 @@ public void SendPacketsElement_FileStreamMultiPart_Success() } } - [Fact] + public static bool IsNotWindows11 = !PlatformDetection.IsWindows10Version22000OrGreater; + + [ActiveIssue("https://github.com/dotnet/runtime/issues/58898")] + [ConditionalFact(nameof(IsNotWindows11))] public void SendPacketsElement_FileStreamLargeOffset_Throws() { using (var stream = new FileStream(TestFileName, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, useAsync: true)) @@ -548,7 +553,8 @@ public void SendPacketsElement_FileStreamLargeCount_Throws() } } - [Fact] + [ActiveIssue("https://github.com/dotnet/runtime/issues/58898")] + [ConditionalFact(nameof(IsNotWindows11))] public void SendPacketsElement_FileStreamWithOptions_Success() { using (var stream = new FileStream(TestFileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 4096, FileOptions.Asynchronous | FileOptions.SequentialScan)) { var element = new SendPacketsElement(stream, 0, s_testFileSize); @@ -579,7 +585,8 @@ public void SendPacketsElement_FileStreamMultiPartMixed_Success() { } } - [Fact] + [ActiveIssue("https://github.com/dotnet/runtime/issues/58898")] + [ConditionalFact(nameof(IsNotWindows11))] public void SendPacketsElement_FileStreamMultiPartMixed_MultipleFileStreams_Success() { using (var stream = new FileStream(TestFileName, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.Asynchronous)) using (var stream2 = new FileStream(TestFileName, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.Asynchronous)) { From 749c02bbfbd75b642da072807978b0c1aee1b321 Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Mon, 10 Jan 2022 09:11:21 -0800 Subject: [PATCH 31/39] [release/6.0] Fix GENERIC_MATH_FEATURE by making sure we always choose the right ref (#63525) --- .../ref/System.Runtime.Experimental.csproj | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Runtime.Experimental/ref/System.Runtime.Experimental.csproj b/src/libraries/System.Runtime.Experimental/ref/System.Runtime.Experimental.csproj index 0ec6e1e5a1e891..b2c0d647ead2a7 100644 --- a/src/libraries/System.Runtime.Experimental/ref/System.Runtime.Experimental.csproj +++ b/src/libraries/System.Runtime.Experimental/ref/System.Runtime.Experimental.csproj @@ -17,6 +17,8 @@ ref $(DefineConstants);FEATURE_GENERIC_MATH true + 1 + true Exposes new experimental APIs from System.Runtime $(MSBuildProjectName) @@ -32,7 +34,7 @@ <_FileVersionMin>$(FileVersion.Split('.')[1]) <_FileVersionBld>$(FileVersion.Split('.')[2]) <_FileVersionRev>$(FileVersion.Split('.')[3]) - $(_FileVersionMaj).$(_FileVersionMin).$([MSBuild]::Add($(_FileVersionBld), 100)).$(_FileVersionRev) + $(_FileVersionMaj).$([MSBuild]::Add($(_FileVersionMin), 100)).$(_FileVersionBld).$(_FileVersionRev) From 7a2649268835f73396748267fed4320a10d9f61f Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Mon, 10 Jan 2022 14:35:08 -0800 Subject: [PATCH 32/39] Fix NETStandard library using JSON source gen (#63520) NETStandard libraries using JSON source gen would fail to load on .NETCore due to missing IsExternalInit in the assembly. On .NETCore this is defined, whereas on .NETStandard JSON carries an internal copy. The compiler emits references to the internal type when a NETStandard library calls init-only setters in JSON types, but these references are not satisfied when the library runs on .NETCore. Fix this by adding a type forward to JSON for the IsExternalInit type. --- ...ystem.Text.Json.Typeforwards.netcoreapp.cs | 6 ++++ .../ref/System.Text.Json.csproj | 5 +++ ...ystem.Text.Json.Typeforwards.netcoreapp.cs | 6 ++++ .../src/System.Text.Json.csproj | 1 + ...em.Text.Json.TestLibrary.Roslyn3.11.csproj | 8 +++++ ...tem.Text.Json.TestLibrary.Roslyn4.0.csproj | 8 +++++ .../System.Text.Json.TestLibrary.targets | 17 ++++++++++ .../TestClasses.cs | 17 ++++++++++ .../NETStandardContextTests.cs | 31 +++++++++++++++++++ ...n.SourceGeneration.Roslyn3.11.Tests.csproj | 1 + ...on.SourceGeneration.Roslyn4.0.Tests.csproj | 1 + ...m.Text.Json.SourceGeneration.Tests.targets | 1 + 12 files changed, 102 insertions(+) create mode 100644 src/libraries/System.Text.Json/ref/System.Text.Json.Typeforwards.netcoreapp.cs create mode 100644 src/libraries/System.Text.Json/src/System.Text.Json.Typeforwards.netcoreapp.cs create mode 100644 src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.TestLibrary/System.Text.Json.TestLibrary.Roslyn3.11.csproj create mode 100644 src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.TestLibrary/System.Text.Json.TestLibrary.Roslyn4.0.csproj create mode 100644 src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.TestLibrary/System.Text.Json.TestLibrary.targets create mode 100644 src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.TestLibrary/TestClasses.cs create mode 100644 src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/NETStandardContextTests.cs diff --git a/src/libraries/System.Text.Json/ref/System.Text.Json.Typeforwards.netcoreapp.cs b/src/libraries/System.Text.Json/ref/System.Text.Json.Typeforwards.netcoreapp.cs new file mode 100644 index 00000000000000..81030616268c3d --- /dev/null +++ b/src/libraries/System.Text.Json/ref/System.Text.Json.Typeforwards.netcoreapp.cs @@ -0,0 +1,6 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// The compiler emits a reference to the internal copy of this type in our non-NETCoreApp assembly +// so we must include a forward to be compatible with libraries compiled against non-NETCoreApp System.Text.Json +[assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.Runtime.CompilerServices.IsExternalInit))] diff --git a/src/libraries/System.Text.Json/ref/System.Text.Json.csproj b/src/libraries/System.Text.Json/ref/System.Text.Json.csproj index 6ef8f2ad1f044c..aa194d3641829b 100644 --- a/src/libraries/System.Text.Json/ref/System.Text.Json.csproj +++ b/src/libraries/System.Text.Json/ref/System.Text.Json.csproj @@ -7,6 +7,11 @@ + + + + + diff --git a/src/libraries/System.Text.Json/src/System.Text.Json.Typeforwards.netcoreapp.cs b/src/libraries/System.Text.Json/src/System.Text.Json.Typeforwards.netcoreapp.cs new file mode 100644 index 00000000000000..81030616268c3d --- /dev/null +++ b/src/libraries/System.Text.Json/src/System.Text.Json.Typeforwards.netcoreapp.cs @@ -0,0 +1,6 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// The compiler emits a reference to the internal copy of this type in our non-NETCoreApp assembly +// so we must include a forward to be compatible with libraries compiled against non-NETCoreApp System.Text.Json +[assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.Runtime.CompilerServices.IsExternalInit))] diff --git a/src/libraries/System.Text.Json/src/System.Text.Json.csproj b/src/libraries/System.Text.Json/src/System.Text.Json.csproj index 3092e2e0b7f991..fbd5fbbb9360aa 100644 --- a/src/libraries/System.Text.Json/src/System.Text.Json.csproj +++ b/src/libraries/System.Text.Json/src/System.Text.Json.csproj @@ -312,6 +312,7 @@ System.Text.Json.Utf8JsonReader + diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.TestLibrary/System.Text.Json.TestLibrary.Roslyn3.11.csproj b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.TestLibrary/System.Text.Json.TestLibrary.Roslyn3.11.csproj new file mode 100644 index 00000000000000..367b28d96d3717 --- /dev/null +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.TestLibrary/System.Text.Json.TestLibrary.Roslyn3.11.csproj @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.TestLibrary/System.Text.Json.TestLibrary.Roslyn4.0.csproj b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.TestLibrary/System.Text.Json.TestLibrary.Roslyn4.0.csproj new file mode 100644 index 00000000000000..138741ea7741a8 --- /dev/null +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.TestLibrary/System.Text.Json.TestLibrary.Roslyn4.0.csproj @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.TestLibrary/System.Text.Json.TestLibrary.targets b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.TestLibrary/System.Text.Json.TestLibrary.targets new file mode 100644 index 00000000000000..bf1d9e18c74a03 --- /dev/null +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.TestLibrary/System.Text.Json.TestLibrary.targets @@ -0,0 +1,17 @@ + + + netstandard2.0 + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.TestLibrary/TestClasses.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.TestLibrary/TestClasses.cs new file mode 100644 index 00000000000000..a0eebf2e4fd383 --- /dev/null +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.TestLibrary/TestClasses.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json.Serialization; + +namespace System.Text.Json.SourceGeneration.Tests.NETStandard +{ + public class MyPoco + { + public string Value { get; set; } + } + + [JsonSerializable(typeof(MyPoco))] + public partial class NETStandardSerializerContext : JsonSerializerContext + { + } +} diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/NETStandardContextTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/NETStandardContextTests.cs new file mode 100644 index 00000000000000..557473b603b7de --- /dev/null +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/NETStandardContextTests.cs @@ -0,0 +1,31 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; +using Xunit; + +namespace System.Text.Json.SourceGeneration.Tests.NETStandard +{ + public class NETStandardContextTests + { + /// + /// Tests that we can serialize and deserialize a type defined in a NETStandard assembly. + /// This tests an issue where we were emitting source-gen logic that caused the compiler + /// to emit a reference to an internal definition of IsExternalInit that was missing + /// on later versions of .NET (since it was defined by the framework). + /// + [Fact] + public void RoundTripNETStandardDefinedSourceGenType() + { + MyPoco expected = new MyPoco() { Value = "Hello from NETStandard type."}; + + string json = JsonSerializer.Serialize(expected, NETStandardSerializerContext.Default.MyPoco); + MyPoco actual = JsonSerializer.Deserialize(json, NETStandardSerializerContext.Default.MyPoco); + Assert.Equal(expected.Value, actual.Value); + } + } +} diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/System.Text.Json.SourceGeneration.Roslyn3.11.Tests.csproj b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/System.Text.Json.SourceGeneration.Roslyn3.11.Tests.csproj index 8df19c3e866071..8ac0061d4094bf 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/System.Text.Json.SourceGeneration.Roslyn3.11.Tests.csproj +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/System.Text.Json.SourceGeneration.Roslyn3.11.Tests.csproj @@ -4,5 +4,6 @@ + diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/System.Text.Json.SourceGeneration.Roslyn4.0.Tests.csproj b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/System.Text.Json.SourceGeneration.Roslyn4.0.Tests.csproj index 1afcea0cc5834c..82a98d9773af10 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/System.Text.Json.SourceGeneration.Roslyn4.0.Tests.csproj +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/System.Text.Json.SourceGeneration.Roslyn4.0.Tests.csproj @@ -4,5 +4,6 @@ + diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/System.Text.Json.SourceGeneration.Tests.targets b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/System.Text.Json.SourceGeneration.Tests.targets index b16e2ccc8f23e2..2f977168e80437 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/System.Text.Json.SourceGeneration.Tests.targets +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/System.Text.Json.SourceGeneration.Tests.targets @@ -74,6 +74,7 @@ + From 3f6d8fada7b05b2792edf0e8e2e446e37b842c53 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 10 Jan 2022 14:35:26 -0800 Subject: [PATCH 33/39] Make delegates unsupported by JsonSerializer (#63514) Co-authored-by: Layomi Akinrinade --- .../gen/JsonSourceGenerator.Parser.cs | 5 ++- .../Value/UnsupportedTypeConverterFactory.cs | 2 ++ .../tests/Common/PropertyVisibilityTests.cs | 32 +++++++++++++++++++ .../Serialization/PropertyVisibilityTests.cs | 4 +++ 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs index 0204a12eed3341..92c60d4ace019c 100644 --- a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs +++ b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Parser.cs @@ -93,6 +93,7 @@ private sealed class Parser private readonly Type? _jsonValueType; // Unsupported types + private readonly Type _delegateType; private readonly Type _typeType; private readonly Type _serializationInfoType; private readonly Type _intPtrType; @@ -221,6 +222,7 @@ public Parser(Compilation compilation, in JsonSourceGenerationContext sourceGene _jsonValueType = _metadataLoadContext.Resolve(JsonValueFullName); // Unsupported types. + _delegateType = _metadataLoadContext.Resolve(SpecialType.System_Delegate); _typeType = _metadataLoadContext.Resolve(typeof(Type)); _serializationInfoType = _metadataLoadContext.Resolve(typeof(Runtime.Serialization.SerializationInfo)); _intPtrType = _metadataLoadContext.Resolve(typeof(IntPtr)); @@ -906,7 +908,8 @@ private TypeGenerationSpec GetOrAddTypeGenerationSpec(Type type, JsonSourceGener } } else if (_knownUnsupportedTypes.Contains(type) || - ImplementsIAsyncEnumerableInterface(type)) + ImplementsIAsyncEnumerableInterface(type) || + _delegateType.IsAssignableFrom(type)) { classType = ClassType.KnownUnsupportedType; } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/UnsupportedTypeConverterFactory.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/UnsupportedTypeConverterFactory.cs index fdc632d96d2f6c..f211939119505b 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/UnsupportedTypeConverterFactory.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/UnsupportedTypeConverterFactory.cs @@ -23,6 +23,8 @@ public override bool CanConvert(Type type) type == typeof(SerializationInfo) || type == typeof(IntPtr) || type == typeof(UIntPtr) || + // Exlude delegates. + typeof(Delegate).IsAssignableFrom(type) || // DateOnly/TimeOnly support to be added in future releases; // guard against invalid object-based serializations for now. // cf. https://github.com/dotnet/runtime/issues/53539 diff --git a/src/libraries/System.Text.Json/tests/Common/PropertyVisibilityTests.cs b/src/libraries/System.Text.Json/tests/Common/PropertyVisibilityTests.cs index a41380e3e60d9c..8c0f57b422a55d 100644 --- a/src/libraries/System.Text.Json/tests/Common/PropertyVisibilityTests.cs +++ b/src/libraries/System.Text.Json/tests/Common/PropertyVisibilityTests.cs @@ -2817,5 +2817,37 @@ public class TypeWith_IgnoredPropWith_BadConverter public class BadConverter { } + + [Fact] + public async Task TestClassWithIgnoredCallbacks() + { + Assert.Equal("{}", await JsonSerializerWrapperForString.SerializeWrapper(new ClassWithIgnoredCallbacks())); + var obj = await JsonSerializerWrapperForString.DeserializeWrapper(@"{""Func"":"""",""Action"":""""}"); + Assert.False(obj.Func("")); + Assert.Null(obj.Action); + } + + [Fact] + public async Task TestClassWithCallbacks() + { + await Assert.ThrowsAsync(async () => await JsonSerializerWrapperForString.SerializeWrapper(new ClassWithCallbacks())); + await Assert.ThrowsAsync(async () => await JsonSerializerWrapperForString.DeserializeWrapper(@"{""Func"":{},""Action"":{}")); + } + + public class ClassWithIgnoredCallbacks + { + [JsonIgnore] + public Func Func { get; set; } = (val) => false; + + [JsonIgnore] + public Action Action { get; set; } + } + + public class ClassWithCallbacks + { + public Func Func { get; set; } + + public Action Action { get; set; } = (val) => Console.WriteLine(); + } } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/Serialization/PropertyVisibilityTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/Serialization/PropertyVisibilityTests.cs index 3b38cefe23e98c..a698165e3f9677 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/Serialization/PropertyVisibilityTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/Serialization/PropertyVisibilityTests.cs @@ -268,6 +268,8 @@ public override async Task HonorJsonPropertyName_PrivateSetter() [JsonSerializable(typeof(TypeWith_IgnoredRefStringProp))] [JsonSerializable(typeof(TypeWith_PropWith_BadConverter))] [JsonSerializable(typeof(TypeWith_IgnoredPropWith_BadConverter))] + [JsonSerializable(typeof(ClassWithIgnoredCallbacks))] + [JsonSerializable(typeof(ClassWithCallbacks))] internal sealed partial class PropertyVisibilityTestsContext_Metadata : JsonSerializerContext { } @@ -439,6 +441,8 @@ public override async Task JsonIgnoreCondition_WhenWritingNull_OnValueType_Fail_ [JsonSerializable(typeof(TypeWith_IgnoredRefStringProp))] [JsonSerializable(typeof(TypeWith_PropWith_BadConverter))] [JsonSerializable(typeof(TypeWith_IgnoredPropWith_BadConverter))] + [JsonSerializable(typeof(ClassWithIgnoredCallbacks))] + [JsonSerializable(typeof(ClassWithCallbacks))] internal sealed partial class PropertyVisibilityTestsContext_Default : JsonSerializerContext { } From 04a58365e6f997cb55c1ab534e5befdf502002e0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 10 Jan 2022 15:32:03 -0800 Subject: [PATCH 34/39] [release/6.0] 1388 xml serializer assembly load context awareness (#61266) * Generate dynamic serialization assembly in the appropriate ALC, and don't keep any hard refs to types that could prevent unloading. * Add small test for ALC unloading. * PR feedback; Move into TempAssembly; Strong/Weak lookup tables; Test refinement * Refining TempAssembly cache; Reflection-based fix; Test corrections * Mono/wasm test fixup * Ensure that serializers from XmlMappings don't run afoul of collectible ALCs. * PR feedback * Unloadable contexts not yet supported in Mono. * Fix trimming of types for XmlSerializer tests that got moved out of main test assembly. * Encapsulating the duality of dealing with unloadable ALC instead of rewriting similar code everywhere. * Missed new file in last commit. * Compilation bug not seen locally. * Perf improvement. * Remove extraneous line. Test feedback. * Fix unloadability (#58948) There is a bug in AppDomain::FindAssembly code path that iterates over the domain assemblies. The iteration loop leaves the refcount of the LoaderAllocator related to the returned DomainAssembly bumped, but nothing ever decrements it. So when a code that needs to be unloaded ends up in that code path, all the managed things like managed LoaderAllocator, LoaderAllocatorScout are destroyed, but the unloading doesn't complete due to the refcount. We have never found it before as this code path is never executed in any of the coreclr tests even with unloadability testing option. (cherry picked from commit 8b38c1973eeaf1c13e8d2729ab1b1fe1a7cf9a5b) Co-authored-by: Steve Molloy Co-authored-by: Jan Vorlicek --- src/coreclr/vm/appdomain.cpp | 4 +- .../System/Runtime/Serialization/Utils.cs | 29 ++ .../src/Resources/Strings.resx | 3 + .../src/System.Private.Xml.csproj | 2 + .../System/Xml/Serialization/Compilation.cs | 309 +++++++++++------- .../Xml/Serialization/ContextAwareTables.cs | 66 ++++ .../ReflectionXmlSerializationReader.cs | 54 +-- .../Serialization/XmlSerializationWriter.cs | 10 +- .../System/Xml/Serialization/XmlSerializer.cs | 60 ++-- ....XmlSerializer.ReflectionOnly.Tests.csproj | 6 +- .../System.Xml.XmlSerializer.Tests.csproj | 6 +- .../tests/XmlSerializer/XmlSerializerTests.cs | 54 ++- .../tests/SerializationTypes.cs | 10 + 13 files changed, 418 insertions(+), 195 deletions(-) create mode 100644 src/libraries/System.Private.Xml/src/System/Xml/Serialization/ContextAwareTables.cs diff --git a/src/coreclr/vm/appdomain.cpp b/src/coreclr/vm/appdomain.cpp index a55090bac1ec5f..13201596b7cfc3 100644 --- a/src/coreclr/vm/appdomain.cpp +++ b/src/coreclr/vm/appdomain.cpp @@ -3533,8 +3533,8 @@ DomainAssembly * AppDomain::FindAssembly(PEAssembly * pFile, FindAssemblyOptions !pManifestFile->IsResource() && pManifestFile->Equals(pFile)) { - // Caller already has PEAssembly, so we can give DomainAssembly away freely without AddRef - return pDomainAssembly.Extract(); + // Caller already has PEAssembly, so we can give DomainAssembly away freely without added reference + return pDomainAssembly.GetValue(); } } return NULL; diff --git a/src/libraries/Common/tests/System/Runtime/Serialization/Utils.cs b/src/libraries/Common/tests/System/Runtime/Serialization/Utils.cs index ca01381a6dff10..2e2ac299c2c53b 100644 --- a/src/libraries/Common/tests/System/Runtime/Serialization/Utils.cs +++ b/src/libraries/Common/tests/System/Runtime/Serialization/Utils.cs @@ -9,6 +9,8 @@ using System.Threading.Tasks; using System.Xml.Linq; using System.Linq; +using System.Reflection; +using System.Runtime.Loader; using Xunit; internal static class Utils @@ -351,3 +353,30 @@ private static bool IsPrefixedAttributeValue(string atrValue, out string localPr return false; } } + +internal class TestAssemblyLoadContext : AssemblyLoadContext +{ + private AssemblyDependencyResolver _resolver; + + public TestAssemblyLoadContext(string name, bool isCollectible, string mainAssemblyToLoadPath = null) : base(name, isCollectible) + { + if (!PlatformDetection.IsBrowser) + _resolver = new AssemblyDependencyResolver(mainAssemblyToLoadPath ?? Assembly.GetExecutingAssembly().Location); + } + + protected override Assembly Load(AssemblyName name) + { + if (PlatformDetection.IsBrowser) + { + return base.Load(name); + } + + string assemblyPath = _resolver.ResolveAssemblyToPath(name); + if (assemblyPath != null) + { + return LoadFromAssemblyPath(assemblyPath); + } + + return null; + } +} diff --git a/src/libraries/System.Private.Xml/src/Resources/Strings.resx b/src/libraries/System.Private.Xml/src/Resources/Strings.resx index d19dc6ec4eb314..112359dfecba8c 100644 --- a/src/libraries/System.Private.Xml/src/Resources/Strings.resx +++ b/src/libraries/System.Private.Xml/src/Resources/Strings.resx @@ -2787,6 +2787,9 @@ Type '{0}' is not serializable. + + Type '{0}' is from an AssemblyLoadContext which is incompatible with that which contains this XmlSerializer. + Invalid XmlSerializerAssemblyAttribute usage. Please use {0} property or {1} property. diff --git a/src/libraries/System.Private.Xml/src/System.Private.Xml.csproj b/src/libraries/System.Private.Xml/src/System.Private.Xml.csproj index 2afc279cf5f805..648245cdb49703 100644 --- a/src/libraries/System.Private.Xml/src/System.Private.Xml.csproj +++ b/src/libraries/System.Private.Xml/src/System.Private.Xml.csproj @@ -446,6 +446,7 @@ + @@ -565,6 +566,7 @@ + diff --git a/src/libraries/System.Private.Xml/src/System/Xml/Serialization/Compilation.cs b/src/libraries/System.Private.Xml/src/System/Xml/Serialization/Compilation.cs index 8fdeed7b60b770..c3df77a5a2bc3f 100644 --- a/src/libraries/System.Private.Xml/src/System/Xml/Serialization/Compilation.cs +++ b/src/libraries/System.Private.Xml/src/System/Xml/Serialization/Compilation.cs @@ -11,6 +11,8 @@ using System.Globalization; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using System.Runtime.Loader; namespace System.Xml.Serialization { @@ -149,79 +151,82 @@ internal void InitAssemblyMethods(XmlMapping[] xmlMappings) contract = null; string? serializerName = null; - // check to see if we loading explicit pre-generated assembly - object[] attrs = type.GetCustomAttributes(typeof(System.Xml.Serialization.XmlSerializerAssemblyAttribute), false); - if (attrs.Length == 0) + using (AssemblyLoadContext.EnterContextualReflection(type.Assembly)) { - // Guess serializer name: if parent assembly signed use strong name - AssemblyName name = type.Assembly.GetName(); - serializerName = Compiler.GetTempAssemblyName(name, defaultNamespace); - // use strong name - name.Name = serializerName; - name.CodeBase = null; - name.CultureInfo = CultureInfo.InvariantCulture; - - try - { - serializer = Assembly.Load(name); - } - catch (Exception e) - { - if (e is OutOfMemoryException) + // check to see if we loading explicit pre-generated assembly + object[] attrs = type.GetCustomAttributes(typeof(System.Xml.Serialization.XmlSerializerAssemblyAttribute), false); + if (attrs.Length == 0) + { + // Guess serializer name: if parent assembly signed use strong name + AssemblyName name = type.Assembly.GetName(); + serializerName = Compiler.GetTempAssemblyName(name, defaultNamespace); + // use strong name + name.Name = serializerName; + name.CodeBase = null; + name.CultureInfo = CultureInfo.InvariantCulture; + + try { - throw; + serializer = Assembly.Load(name); } - } - - serializer ??= LoadAssemblyByPath(type, serializerName); - - if (serializer == null) - { - if (XmlSerializer.Mode == SerializationMode.PreGenOnly) + catch (Exception e) { - throw new Exception(SR.Format(SR.FailLoadAssemblyUnderPregenMode, serializerName)); + if (e is OutOfMemoryException) + { + throw; + } } - return null; - } + serializer ??= LoadAssemblyByPath(type, serializerName); - if (!IsSerializerVersionMatch(serializer, type, defaultNamespace)) - { - XmlSerializationEventSource.Log.XmlSerializerExpired(serializerName, type.FullName!); - return null; - } - } - else - { - System.Xml.Serialization.XmlSerializerAssemblyAttribute assemblyAttribute = (System.Xml.Serialization.XmlSerializerAssemblyAttribute)attrs[0]; - if (assemblyAttribute.AssemblyName != null && assemblyAttribute.CodeBase != null) - throw new InvalidOperationException(SR.Format(SR.XmlPregenInvalidXmlSerializerAssemblyAttribute, "AssemblyName", "CodeBase")); + if (serializer == null) + { + if (XmlSerializer.Mode == SerializationMode.PreGenOnly) + { + throw new Exception(SR.Format(SR.FailLoadAssemblyUnderPregenMode, serializerName)); + } - // found XmlSerializerAssemblyAttribute attribute, it should have all needed information to load the pre-generated serializer - if (assemblyAttribute.AssemblyName != null) - { - serializerName = assemblyAttribute.AssemblyName; - serializer = Assembly.Load(serializerName); // LoadWithPartialName just does this in .Net Core; changing the obsolete call. - } - else if (assemblyAttribute.CodeBase != null && assemblyAttribute.CodeBase.Length > 0) - { - serializerName = assemblyAttribute.CodeBase; - serializer = Assembly.LoadFrom(serializerName); + return null; + } + + if (!IsSerializerVersionMatch(serializer, type, defaultNamespace)) + { + XmlSerializationEventSource.Log.XmlSerializerExpired(serializerName, type.FullName!); + return null; + } } else { - serializerName = type.Assembly.FullName; - serializer = type.Assembly; - } - if (serializer == null) - { - throw new FileNotFoundException(null, serializerName); + System.Xml.Serialization.XmlSerializerAssemblyAttribute assemblyAttribute = (System.Xml.Serialization.XmlSerializerAssemblyAttribute)attrs[0]; + if (assemblyAttribute.AssemblyName != null && assemblyAttribute.CodeBase != null) + throw new InvalidOperationException(SR.Format(SR.XmlPregenInvalidXmlSerializerAssemblyAttribute, "AssemblyName", "CodeBase")); + + // found XmlSerializerAssemblyAttribute attribute, it should have all needed information to load the pre-generated serializer + if (assemblyAttribute.AssemblyName != null) + { + serializerName = assemblyAttribute.AssemblyName; + serializer = Assembly.Load(serializerName); // LoadWithPartialName just does this in .Net Core; changing the obsolete call. + } + else if (assemblyAttribute.CodeBase != null && assemblyAttribute.CodeBase.Length > 0) + { + serializerName = assemblyAttribute.CodeBase; + serializer = Assembly.LoadFrom(serializerName); + } + else + { + serializerName = type.Assembly.FullName; + serializer = type.Assembly; + } + if (serializer == null) + { + throw new FileNotFoundException(null, serializerName); + } } + Type contractType = GetTypeFromAssembly(serializer, "XmlSerializerContract"); + contract = (XmlSerializerImplementation)Activator.CreateInstance(contractType)!; + if (contract.CanSerialize(type)) + return serializer; } - Type contractType = GetTypeFromAssembly(serializer, "XmlSerializerContract"); - contract = (XmlSerializerImplementation)Activator.CreateInstance(contractType)!; - if (contract.CanSerialize(type)) - return serializer; return null; } @@ -449,81 +454,93 @@ internal static bool GenerateSerializerToStream(XmlMapping[] xmlMappings, Type?[ } [RequiresUnreferencedCode("calls GenerateElement")] - internal static Assembly GenerateRefEmitAssembly(XmlMapping[] xmlMappings, Type?[]? types, string? defaultNamespace) + internal static Assembly GenerateRefEmitAssembly(XmlMapping[] xmlMappings, Type?[] types, string? defaultNamespace) { + var mainType = (types.Length > 0) ? types[0] : null; + Assembly? mainAssembly = mainType?.Assembly; var scopeTable = new Dictionary(); foreach (XmlMapping mapping in xmlMappings) scopeTable[mapping.Scope!] = mapping; TypeScope[] scopes = new TypeScope[scopeTable.Keys.Count]; scopeTable.Keys.CopyTo(scopes, 0); - string assemblyName = "Microsoft.GeneratedCode"; - AssemblyBuilder assemblyBuilder = CodeGenerator.CreateAssemblyBuilder(assemblyName); - // Add AssemblyVersion attribute to match parent assembly version - if (types != null && types.Length > 0 && types[0] != null) + using (AssemblyLoadContext.EnterContextualReflection(mainAssembly)) { - ConstructorInfo AssemblyVersionAttribute_ctor = typeof(AssemblyVersionAttribute).GetConstructor( - new Type[] { typeof(string) } - )!; - string assemblyVersion = types[0]!.Assembly.GetName().Version!.ToString(); - assemblyBuilder.SetCustomAttribute(new CustomAttributeBuilder(AssemblyVersionAttribute_ctor, new object[] { assemblyVersion })); - } - CodeIdentifiers classes = new CodeIdentifiers(); - classes.AddUnique("XmlSerializationWriter", "XmlSerializationWriter"); - classes.AddUnique("XmlSerializationReader", "XmlSerializationReader"); - string? suffix = null; - if (types != null && types.Length == 1 && types[0] != null) - { - suffix = CodeIdentifier.MakeValid(types[0]!.Name); - if (types[0]!.IsArray) + // Before generating any IL, check each mapping and supported type to make sure + // they are compatible with the current ALC + for (int i = 0; i < types.Length; i++) + VerifyLoadContext(types[i], mainAssembly); + foreach (var mapping in xmlMappings) + VerifyLoadContext(mapping.Accessor.Mapping?.TypeDesc?.Type, mainAssembly); + + string assemblyName = "Microsoft.GeneratedCode"; + AssemblyBuilder assemblyBuilder = CodeGenerator.CreateAssemblyBuilder(assemblyName); + // Add AssemblyVersion attribute to match parent assembly version + if (mainType != null) + { + ConstructorInfo AssemblyVersionAttribute_ctor = typeof(AssemblyVersionAttribute).GetConstructor( + new Type[] { typeof(string) } + )!; + string assemblyVersion = mainType.Assembly.GetName().Version!.ToString(); + assemblyBuilder.SetCustomAttribute(new CustomAttributeBuilder(AssemblyVersionAttribute_ctor, new object[] { assemblyVersion })); + } + CodeIdentifiers classes = new CodeIdentifiers(); + classes.AddUnique("XmlSerializationWriter", "XmlSerializationWriter"); + classes.AddUnique("XmlSerializationReader", "XmlSerializationReader"); + string? suffix = null; + if (mainType != null) { - suffix += "Array"; + suffix = CodeIdentifier.MakeValid(mainType.Name); + if (mainType.IsArray) + { + suffix += "Array"; + } } - } - ModuleBuilder moduleBuilder = CodeGenerator.CreateModuleBuilder(assemblyBuilder, assemblyName); + ModuleBuilder moduleBuilder = CodeGenerator.CreateModuleBuilder(assemblyBuilder, assemblyName); - string writerClass = "XmlSerializationWriter" + suffix; - writerClass = classes.AddUnique(writerClass, writerClass); - XmlSerializationWriterILGen writerCodeGen = new XmlSerializationWriterILGen(scopes, "public", writerClass); - writerCodeGen.ModuleBuilder = moduleBuilder; + string writerClass = "XmlSerializationWriter" + suffix; + writerClass = classes.AddUnique(writerClass, writerClass); + XmlSerializationWriterILGen writerCodeGen = new XmlSerializationWriterILGen(scopes, "public", writerClass); + writerCodeGen.ModuleBuilder = moduleBuilder; - writerCodeGen.GenerateBegin(); - string[] writeMethodNames = new string[xmlMappings.Length]; + writerCodeGen.GenerateBegin(); + string[] writeMethodNames = new string[xmlMappings.Length]; - for (int i = 0; i < xmlMappings.Length; i++) - { - writeMethodNames[i] = writerCodeGen.GenerateElement(xmlMappings[i])!; - } - Type writerType = writerCodeGen.GenerateEnd(); + for (int i = 0; i < xmlMappings.Length; i++) + { + writeMethodNames[i] = writerCodeGen.GenerateElement(xmlMappings[i])!; + } + Type writerType = writerCodeGen.GenerateEnd(); - string readerClass = "XmlSerializationReader" + suffix; - readerClass = classes.AddUnique(readerClass, readerClass); - XmlSerializationReaderILGen readerCodeGen = new XmlSerializationReaderILGen(scopes, "public", readerClass); + string readerClass = "XmlSerializationReader" + suffix; + readerClass = classes.AddUnique(readerClass, readerClass); + XmlSerializationReaderILGen readerCodeGen = new XmlSerializationReaderILGen(scopes, "public", readerClass); - readerCodeGen.ModuleBuilder = moduleBuilder; - readerCodeGen.CreatedTypes.Add(writerType.Name, writerType); + readerCodeGen.ModuleBuilder = moduleBuilder; + readerCodeGen.CreatedTypes.Add(writerType.Name, writerType); - readerCodeGen.GenerateBegin(); - string[] readMethodNames = new string[xmlMappings.Length]; - for (int i = 0; i < xmlMappings.Length; i++) - { - readMethodNames[i] = readerCodeGen.GenerateElement(xmlMappings[i])!; - } - readerCodeGen.GenerateEnd(readMethodNames, xmlMappings, types!); + readerCodeGen.GenerateBegin(); + string[] readMethodNames = new string[xmlMappings.Length]; + for (int i = 0; i < xmlMappings.Length; i++) + { + readMethodNames[i] = readerCodeGen.GenerateElement(xmlMappings[i])!; + } + readerCodeGen.GenerateEnd(readMethodNames, xmlMappings, types!); - string baseSerializer = readerCodeGen.GenerateBaseSerializer("XmlSerializer1", readerClass, writerClass, classes); - var serializers = new Dictionary(); - for (int i = 0; i < xmlMappings.Length; i++) - { - if (!serializers.ContainsKey(xmlMappings[i].Key!)) + string baseSerializer = readerCodeGen.GenerateBaseSerializer("XmlSerializer1", readerClass, writerClass, classes); + var serializers = new Dictionary(); + for (int i = 0; i < xmlMappings.Length; i++) { - serializers[xmlMappings[i].Key!] = readerCodeGen.GenerateTypedSerializer(readMethodNames[i], writeMethodNames[i], xmlMappings[i], classes, baseSerializer, readerClass, writerClass); + if (!serializers.ContainsKey(xmlMappings[i].Key!)) + { + serializers[xmlMappings[i].Key!] = readerCodeGen.GenerateTypedSerializer(readMethodNames[i], writeMethodNames[i], xmlMappings[i], classes, baseSerializer, readerClass, writerClass); + } } - } - readerCodeGen.GenerateSerializerContract("XmlSerializerContract", xmlMappings, types!, readerClass, readMethodNames, writerClass, writeMethodNames, serializers); + readerCodeGen.GenerateSerializerContract("XmlSerializerContract", xmlMappings, types!, readerClass, readMethodNames, writerClass, writeMethodNames, serializers); - return writerType.Assembly; + return writerType.Assembly; + } } private static MethodInfo GetMethodFromType( @@ -588,6 +605,23 @@ internal bool CanRead(XmlMapping mapping, XmlReader xmlReader) return encodingStyle; } + internal static void VerifyLoadContext(Type? t, Assembly? assembly) + { + // The quick case, t is null or in the same assembly + if (t == null || assembly == null || t.Assembly == assembly) + return; + + // No worries if the type is not collectible + var typeALC = AssemblyLoadContext.GetLoadContext(t.Assembly); + if (typeALC == null || !typeALC.IsCollectible) + return; + + // Collectible types should be in the same collectible context + var baseALC = AssemblyLoadContext.GetLoadContext(assembly) ?? AssemblyLoadContext.CurrentContextualReflectionContext; + if (typeALC != baseALC) + throw new InvalidOperationException(SR.Format(SR.XmlTypeInBadLoadContext, t.FullName)); + } + [RequiresUnreferencedCode("calls Contract")] internal object? InvokeReader(XmlMapping mapping, XmlReader xmlReader, XmlDeserializationEvents events, string? encodingStyle) { @@ -666,9 +700,9 @@ internal sealed class TempMethodDictionary : Dictionary internal sealed class TempAssemblyCacheKey { private readonly string? _ns; - private readonly object _type; + private readonly Type _type; - internal TempAssemblyCacheKey(string? ns, object type) + internal TempAssemblyCacheKey(string? ns, Type type) { _type = type; _ns = ns; @@ -690,29 +724,52 @@ public override int GetHashCode() internal sealed class TempAssemblyCache { - private Dictionary _cache = new Dictionary(); + private Dictionary _fastCache = new Dictionary(); + private ConditionalWeakTable> _collectibleCaches = new ConditionalWeakTable>(); - internal TempAssembly? this[string? ns, object o] + internal TempAssembly? this[string? ns, Type t] { get { TempAssembly? tempAssembly; - _cache.TryGetValue(new TempAssemblyCacheKey(ns, o), out tempAssembly); + TempAssemblyCacheKey key = new TempAssemblyCacheKey(ns, t); + + if (_fastCache.TryGetValue(key, out tempAssembly)) + return tempAssembly; + + if (_collectibleCaches.TryGetValue(t.Assembly, out var cCache)) + cCache.TryGetValue(key, out tempAssembly); + return tempAssembly; } } - internal void Add(string? ns, object o, TempAssembly assembly) + internal void Add(string? ns, Type t, TempAssembly assembly) { - TempAssemblyCacheKey key = new TempAssemblyCacheKey(ns, o); lock (this) { - TempAssembly? tempAssembly; - if (_cache.TryGetValue(key, out tempAssembly) && tempAssembly == assembly) + TempAssembly? tempAssembly = this[ns, t]; + if (tempAssembly == assembly) return; - Dictionary _copy = new Dictionary(_cache); // clone - _copy[key] = assembly; - _cache = _copy; + + AssemblyLoadContext? alc = AssemblyLoadContext.GetLoadContext(t.Assembly); + TempAssemblyCacheKey key = new TempAssemblyCacheKey(ns, t); + Dictionary? cache; + + if (alc != null && alc.IsCollectible) + { + cache = _collectibleCaches.TryGetValue(t.Assembly, out var c) // Clone or create + ? new Dictionary(c) + : new Dictionary(); + cache[key] = assembly; + _collectibleCaches.AddOrUpdate(t.Assembly, cache); + } + else + { + cache = new Dictionary(_fastCache); // Clone + cache[key] = assembly; + _fastCache = cache; + } } } } diff --git a/src/libraries/System.Private.Xml/src/System/Xml/Serialization/ContextAwareTables.cs b/src/libraries/System.Private.Xml/src/System/Xml/Serialization/ContextAwareTables.cs new file mode 100644 index 00000000000000..64e09bbc063b2c --- /dev/null +++ b/src/libraries/System.Private.Xml/src/System/Xml/Serialization/ContextAwareTables.cs @@ -0,0 +1,66 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System.Xml.Serialization +{ + using System; + using System.Collections; + using System.Diagnostics.CodeAnalysis; + using System.Runtime.CompilerServices; + using System.Runtime.Loader; + + internal class ContextAwareTables<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicParameterlessConstructor)]T> where T : class? + { + private Hashtable _defaultTable; + private ConditionalWeakTable _collectibleTable; + + public ContextAwareTables() + { + _defaultTable = new Hashtable(); + _collectibleTable = new ConditionalWeakTable(); + } + + internal T GetOrCreateValue(Type t, Func f) + { + // The fast and most common default case + T? ret = (T?)_defaultTable[t]; + if (ret != null) + return ret; + + // Common case for collectible contexts + if (_collectibleTable.TryGetValue(t, out ret)) + return ret; + + // Not found. Do the slower work of creating the value in the correct collection. + AssemblyLoadContext? alc = AssemblyLoadContext.GetLoadContext(t.Assembly); + + // Null and non-collectible load contexts use the default table + if (alc == null || !alc.IsCollectible) + { + lock (_defaultTable) + { + if ((ret = (T?)_defaultTable[t]) == null) + { + ret = f(); + _defaultTable[t] = ret; + } + } + } + + // Collectible load contexts should use the ConditionalWeakTable so they can be unloaded + else + { + lock (_collectibleTable) + { + if (!_collectibleTable.TryGetValue(t, out ret)) + { + ret = f(); + _collectibleTable.AddOrUpdate(t, ret); + } + } + } + + return ret; + } + } +} diff --git a/src/libraries/System.Private.Xml/src/System/Xml/Serialization/ReflectionXmlSerializationReader.cs b/src/libraries/System.Private.Xml/src/System/Xml/Serialization/ReflectionXmlSerializationReader.cs index 0c55638a27a66a..2477f283b01f16 100644 --- a/src/libraries/System.Private.Xml/src/System/Xml/Serialization/ReflectionXmlSerializationReader.cs +++ b/src/libraries/System.Private.Xml/src/System/Xml/Serialization/ReflectionXmlSerializationReader.cs @@ -627,40 +627,48 @@ private static void AddObjectsIntoTargetCollection(object targetCollection, List } } - private static readonly ConcurrentDictionary<(Type, string), ReflectionXmlSerializationReaderHelper.SetMemberValueDelegate> s_setMemberValueDelegateCache = new ConcurrentDictionary<(Type, string), ReflectionXmlSerializationReaderHelper.SetMemberValueDelegate>(); + private static readonly ContextAwareTables s_setMemberValueDelegateCache = new ContextAwareTables(); [RequiresUnreferencedCode(XmlSerializer.TrimSerializationWarning)] private static ReflectionXmlSerializationReaderHelper.SetMemberValueDelegate GetSetMemberValueDelegate(object o, string memberName) { Debug.Assert(o != null, "Object o should not be null"); Debug.Assert(!string.IsNullOrEmpty(memberName), "memberName must have a value"); - (Type, string) typeMemberNameTuple = (o.GetType(), memberName); - if (!s_setMemberValueDelegateCache.TryGetValue(typeMemberNameTuple, out ReflectionXmlSerializationReaderHelper.SetMemberValueDelegate? result)) + Type type = o.GetType(); + var delegateCacheForType = s_setMemberValueDelegateCache.GetOrCreateValue(type, () => new Hashtable()); + var result = delegateCacheForType[memberName]; + if (result == null) { - MemberInfo memberInfo = ReflectionXmlSerializationHelper.GetEffectiveSetInfo(o.GetType(), memberName); - Debug.Assert(memberInfo != null, "memberInfo could not be retrieved"); - Type memberType; - if (memberInfo is PropertyInfo propInfo) - { - memberType = propInfo.PropertyType; - } - else if (memberInfo is FieldInfo fieldInfo) - { - memberType = fieldInfo.FieldType; - } - else + lock (delegateCacheForType) { - throw new InvalidOperationException(SR.XmlInternalError); - } + if ((result = delegateCacheForType[memberName]) == null) + { + MemberInfo memberInfo = ReflectionXmlSerializationHelper.GetEffectiveSetInfo(o.GetType(), memberName); + Debug.Assert(memberInfo != null, "memberInfo could not be retrieved"); + Type memberType; + if (memberInfo is PropertyInfo propInfo) + { + memberType = propInfo.PropertyType; + } + else if (memberInfo is FieldInfo fieldInfo) + { + memberType = fieldInfo.FieldType; + } + else + { + throw new InvalidOperationException(SR.XmlInternalError); + } - MethodInfo getSetMemberValueDelegateWithTypeGenericMi = typeof(ReflectionXmlSerializationReaderHelper).GetMethod("GetSetMemberValueDelegateWithType", BindingFlags.Static | BindingFlags.Public)!; - MethodInfo getSetMemberValueDelegateWithTypeMi = getSetMemberValueDelegateWithTypeGenericMi.MakeGenericMethod(o.GetType(), memberType); - var getSetMemberValueDelegateWithType = (Func)getSetMemberValueDelegateWithTypeMi.CreateDelegate(typeof(Func)); - result = getSetMemberValueDelegateWithType(memberInfo); - s_setMemberValueDelegateCache.TryAdd(typeMemberNameTuple, result); + MethodInfo getSetMemberValueDelegateWithTypeGenericMi = typeof(ReflectionXmlSerializationReaderHelper).GetMethod("GetSetMemberValueDelegateWithType", BindingFlags.Static | BindingFlags.Public)!; + MethodInfo getSetMemberValueDelegateWithTypeMi = getSetMemberValueDelegateWithTypeGenericMi.MakeGenericMethod(o.GetType(), memberType); + var getSetMemberValueDelegateWithType = (Func)getSetMemberValueDelegateWithTypeMi.CreateDelegate(typeof(Func)); + result = getSetMemberValueDelegateWithType(memberInfo); + delegateCacheForType[memberName] = result; + } + } } - return result; + return (ReflectionXmlSerializationReaderHelper.SetMemberValueDelegate)result; } private object? GetMemberValue(object o, MemberInfo memberInfo) diff --git a/src/libraries/System.Private.Xml/src/System/Xml/Serialization/XmlSerializationWriter.cs b/src/libraries/System.Private.Xml/src/System/Xml/Serialization/XmlSerializationWriter.cs index 46877f900e9028..7ebc4462dfb7a6 100644 --- a/src/libraries/System.Private.Xml/src/System/Xml/Serialization/XmlSerializationWriter.cs +++ b/src/libraries/System.Private.Xml/src/System/Xml/Serialization/XmlSerializationWriter.cs @@ -19,6 +19,7 @@ namespace System.Xml.Serialization using System.Xml.Serialization; using System.Xml; using System.Diagnostics.CodeAnalysis; + using System.Runtime.CompilerServices; /// public abstract class XmlSerializationWriter : XmlSerializationGeneratedCode @@ -1465,14 +1466,13 @@ internal static class DynamicAssemblies { private static readonly Hashtable s_nameToAssemblyMap = new Hashtable(); private static readonly Hashtable s_assemblyToNameMap = new Hashtable(); - private static readonly Hashtable s_tableIsTypeDynamic = Hashtable.Synchronized(new Hashtable()); + private static readonly ContextAwareTables s_tableIsTypeDynamic = new ContextAwareTables(); // SxS: This method does not take any resource name and does not expose any resources to the caller. // It's OK to suppress the SxS warning. internal static bool IsTypeDynamic(Type type) { - object? oIsTypeDynamic = s_tableIsTypeDynamic[type]; - if (oIsTypeDynamic == null) + object oIsTypeDynamic = s_tableIsTypeDynamic.GetOrCreateValue(type, () => { Assembly assembly = type.Assembly; bool isTypeDynamic = assembly.IsDynamic /*|| string.IsNullOrEmpty(assembly.Location)*/; @@ -1500,8 +1500,8 @@ internal static bool IsTypeDynamic(Type type) } } } - s_tableIsTypeDynamic[type] = oIsTypeDynamic = isTypeDynamic; - } + return isTypeDynamic; + }); return (bool)oIsTypeDynamic; } diff --git a/src/libraries/System.Private.Xml/src/System/Xml/Serialization/XmlSerializer.cs b/src/libraries/System.Private.Xml/src/System/Xml/Serialization/XmlSerializer.cs index 20f9d18343cbb8..b920b8f64a1cf6 100644 --- a/src/libraries/System.Private.Xml/src/System/Xml/Serialization/XmlSerializer.cs +++ b/src/libraries/System.Private.Xml/src/System/Xml/Serialization/XmlSerializer.cs @@ -10,6 +10,7 @@ using System.IO; using System.Reflection; using System.Runtime.CompilerServices; +using System.Runtime.Loader; using System.Runtime.Versioning; using System.Security; using System.Text; @@ -161,8 +162,7 @@ private static XmlSerializerNamespaces DefaultNamespaces internal const string TrimSerializationWarning = "Members from serialized types may be trimmed if not referenced directly"; private const string TrimDeserializationWarning = "Members from deserialized types may be trimmed if not referenced directly"; - private static readonly Dictionary> s_xmlSerializerTable = new Dictionary>(); - + private static readonly ContextAwareTables> s_xmlSerializerTable = new ContextAwareTables>(); protected XmlSerializer() { } @@ -235,30 +235,28 @@ public XmlSerializer(Type type, string? defaultNamespace) _tempAssembly = s_cache[defaultNamespace, type]; if (_tempAssembly == null) { + XmlSerializerImplementation? contract = null; + Assembly? assembly = TempAssembly.LoadGeneratedAssembly(type, defaultNamespace, out contract); + if (assembly == null) { - XmlSerializerImplementation? contract = null; - Assembly? assembly = TempAssembly.LoadGeneratedAssembly(type, defaultNamespace, out contract); - if (assembly == null) - { - if (Mode == SerializationMode.PreGenOnly) - { - AssemblyName name = type.Assembly.GetName(); - var serializerName = Compiler.GetTempAssemblyName(name, defaultNamespace); - throw new FileLoadException(SR.Format(SR.FailLoadAssemblyUnderPregenMode, serializerName)); - } - - // need to reflect and generate new serialization assembly - XmlReflectionImporter importer = new XmlReflectionImporter(defaultNamespace); - _mapping = importer.ImportTypeMapping(type, null, defaultNamespace); - _tempAssembly = GenerateTempAssembly(_mapping, type, defaultNamespace)!; - } - else + if (Mode == SerializationMode.PreGenOnly) { - // we found the pre-generated assembly, now make sure that the assembly has the right serializer - // try to avoid the reflection step, need to get ElementName, namespace and the Key form the type - _mapping = XmlReflectionImporter.GetTopLevelMapping(type, defaultNamespace); - _tempAssembly = new TempAssembly(new XmlMapping[] { _mapping }, assembly, contract); + AssemblyName name = type.Assembly.GetName(); + var serializerName = Compiler.GetTempAssemblyName(name, defaultNamespace); + throw new FileLoadException(SR.Format(SR.FailLoadAssemblyUnderPregenMode, serializerName)); } + + // need to reflect and generate new serialization assembly + XmlReflectionImporter importer = new XmlReflectionImporter(defaultNamespace); + _mapping = importer.ImportTypeMapping(type, null, defaultNamespace); + _tempAssembly = GenerateTempAssembly(_mapping, type, defaultNamespace)!; + } + else + { + // we found the pre-generated assembly, now make sure that the assembly has the right serializer + // try to avoid the reflection step, need to get ElementName, namespace and the Key form the type + _mapping = XmlReflectionImporter.GetTopLevelMapping(type, defaultNamespace); + _tempAssembly = new TempAssembly(new XmlMapping[] { _mapping }, assembly, contract); } } s_cache.Add(defaultNamespace, type, _tempAssembly); @@ -403,7 +401,9 @@ public void Serialize(XmlWriter xmlWriter, object? o, XmlSerializerNamespaces? n } } else + { _tempAssembly.InvokeWriter(_mapping, xmlWriter, o, namespaces == null || namespaces.Count == 0 ? DefaultNamespaces : namespaces, encodingStyle, id); + } } catch (Exception? e) { @@ -629,7 +629,10 @@ public static XmlSerializer[] FromMappings(XmlMapping[]? mappings, Type? type) { XmlSerializer[] serializers = new XmlSerializer[mappings.Length]; for (int i = 0; i < serializers.Length; i++) + { serializers[i] = (XmlSerializer)contract!.TypedSerializers[mappings[i].Key!]!; + TempAssembly.VerifyLoadContext(serializers[i]._rootType, type!.Assembly); + } return serializers; } } @@ -696,16 +699,9 @@ internal static bool GenerateSerializer(Type[]? types, XmlMapping[] mappings, St private static XmlSerializer[] GetSerializersFromCache(XmlMapping[] mappings, Type type) { XmlSerializer?[] serializers = new XmlSerializer?[mappings.Length]; - Dictionary? typedMappingTable = null; - lock (s_xmlSerializerTable) - { - if (!s_xmlSerializerTable.TryGetValue(type, out typedMappingTable)) - { - typedMappingTable = new Dictionary(); - s_xmlSerializerTable[type] = typedMappingTable; - } - } + + typedMappingTable = s_xmlSerializerTable.GetOrCreateValue(type, () => new Dictionary()); lock (typedMappingTable) { diff --git a/src/libraries/System.Private.Xml/tests/XmlSerializer/ReflectionOnly/System.Xml.XmlSerializer.ReflectionOnly.Tests.csproj b/src/libraries/System.Private.Xml/tests/XmlSerializer/ReflectionOnly/System.Xml.XmlSerializer.ReflectionOnly.Tests.csproj index 53abcbd3957022..d7d1f3af3bb87f 100644 --- a/src/libraries/System.Private.Xml/tests/XmlSerializer/ReflectionOnly/System.Xml.XmlSerializer.ReflectionOnly.Tests.csproj +++ b/src/libraries/System.Private.Xml/tests/XmlSerializer/ReflectionOnly/System.Xml.XmlSerializer.ReflectionOnly.Tests.csproj @@ -3,9 +3,13 @@ $(DefineConstants);ReflectionOnly $(NetCoreAppCurrent) + + + + - + diff --git a/src/libraries/System.Private.Xml/tests/XmlSerializer/System.Xml.XmlSerializer.Tests.csproj b/src/libraries/System.Private.Xml/tests/XmlSerializer/System.Xml.XmlSerializer.Tests.csproj index dbc8447b87c1cb..7818cd132825c4 100644 --- a/src/libraries/System.Private.Xml/tests/XmlSerializer/System.Xml.XmlSerializer.Tests.csproj +++ b/src/libraries/System.Private.Xml/tests/XmlSerializer/System.Xml.XmlSerializer.Tests.csproj @@ -2,10 +2,14 @@ $(NetCoreAppCurrent) + + + + - + diff --git a/src/libraries/System.Private.Xml/tests/XmlSerializer/XmlSerializerTests.cs b/src/libraries/System.Private.Xml/tests/XmlSerializer/XmlSerializerTests.cs index c918520a19c8ff..9fefff2137faa5 100644 --- a/src/libraries/System.Private.Xml/tests/XmlSerializer/XmlSerializerTests.cs +++ b/src/libraries/System.Private.Xml/tests/XmlSerializer/XmlSerializerTests.cs @@ -8,6 +8,8 @@ using System.IO; using System.Linq; using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.Loader; using System.Text; using System.Threading; using System.Xml; @@ -1960,6 +1962,52 @@ public static void Xml_TypeWithSpecialCharacterInStringMember() Assert.Equal(x.Name, y.Name); } + [Fact] +#if XMLSERIALIZERGENERATORTESTS + // Lack of AssemblyDependencyResolver results in assemblies that are not loaded by path to get + // loaded in the default ALC, which causes problems for this test. + [SkipOnPlatform(TestPlatforms.Browser, "AssemblyDependencyResolver not supported in wasm")] +#endif + [ActiveIssue("34072", TestRuntimes.Mono)] + public static void Xml_TypeInCollectibleALC() + { + ExecuteAndUnload("SerializableAssembly.dll", "SerializationTypes.SimpleType", out var weakRef); + + for (int i = 0; weakRef.IsAlive && i < 10; i++) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + } + Assert.True(!weakRef.IsAlive); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void ExecuteAndUnload(string assemblyfile, string typename, out WeakReference wref) + { + var fullPath = Path.GetFullPath(assemblyfile); + var alc = new TestAssemblyLoadContext("XmlSerializerTests", true, fullPath); + wref = new WeakReference(alc); + + // Load assembly by path. By name, and it gets loaded in the default ALC. + var asm = alc.LoadFromAssemblyPath(fullPath); + + // Ensure the type loaded in the intended non-Default ALC + var type = asm.GetType(typename); + Assert.Equal(AssemblyLoadContext.GetLoadContext(type.Assembly), alc); + Assert.NotEqual(alc, AssemblyLoadContext.Default); + + // Round-Trip the instance + XmlSerializer serializer = new XmlSerializer(type); + var obj = Activator.CreateInstance(type); + var rtobj = SerializeAndDeserialize(obj, null, () => serializer, true); + Assert.NotNull(rtobj); + Assert.True(rtobj.Equals(obj)); + Assert.Equal(AssemblyLoadContext.GetLoadContext(rtobj.GetType().Assembly), alc); + + alc.Unload(); + } + + private static readonly string s_defaultNs = "http://tempuri.org/"; private static T RoundTripWithXmlMembersMapping(object requestBodyValue, string memberName, string baseline, bool skipStringCompare = false, string wrapperName = null) { @@ -2080,11 +2128,7 @@ private static Stream GenerateStreamFromString(string s) private static T SerializeAndDeserialize(T value, string baseline, Func serializerFactory = null, bool skipStringCompare = false, XmlSerializerNamespaces xns = null) { - XmlSerializer serializer = new XmlSerializer(typeof(T)); - if (serializerFactory != null) - { - serializer = serializerFactory(); - } + XmlSerializer serializer = (serializerFactory != null) ? serializerFactory() : new XmlSerializer(typeof(T)); using (MemoryStream ms = new MemoryStream()) { diff --git a/src/libraries/System.Runtime.Serialization.Xml/tests/SerializationTypes.cs b/src/libraries/System.Runtime.Serialization.Xml/tests/SerializationTypes.cs index bc10a370b033cd..747f661129a94a 100644 --- a/src/libraries/System.Runtime.Serialization.Xml/tests/SerializationTypes.cs +++ b/src/libraries/System.Runtime.Serialization.Xml/tests/SerializationTypes.cs @@ -47,6 +47,16 @@ public static bool AreEqual(SimpleType x, SimpleType y) return (x.P1 == y.P1) && (x.P2 == y.P2); } } + + public override bool Equals(object? obj) + { + if (obj is SimpleType st) + return AreEqual(this, st); + + return base.Equals(obj); + } + + public override int GetHashCode() => base.GetHashCode(); } public class TypeWithGetSetArrayMembers From 7780d2819d2709d61bc03e417a8bbfed6d8b37ed Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 10 Jan 2022 15:34:53 -0800 Subject: [PATCH 35/39] Fix custom JsonConverterFactories not working with transitive type/property declarations. (#62643) Co-authored-by: Eirik Tsarpalis --- .../gen/JsonSourceGenerator.Emitter.cs | 24 +++++------ .../ContextClasses.cs | 4 +- .../JsonSerializerContextTests.cs | 40 +++++++++++++++++++ .../MetadataAndSerializationContextTests.cs | 8 ++-- .../MetadataContextTests.cs | 16 ++++---- .../MixedModeContextTests.cs | 8 ++-- .../RealWorldContextTests.cs | 20 +++++----- .../SerializationContextTests.cs | 16 ++++---- .../TestClasses.CustomConverters.cs | 4 +- 9 files changed, 90 insertions(+), 50 deletions(-) diff --git a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs index 706f42be018bbf..0df820dcecf22e 100644 --- a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs +++ b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs @@ -98,6 +98,9 @@ private sealed partial class Emitter private readonly HashSet _emittedPropertyFileNames = new(); + private bool _generateGetConverterMethodForTypes = false; + private bool _generateGetConverterMethodForProperties = false; + public Emitter(in JsonSourceGenerationContext sourceGenerationContext, SourceGenerationSpec generationSpec) { _sourceGenerationContext = sourceGenerationContext; @@ -109,16 +112,12 @@ public void Emit() foreach (ContextGenerationSpec contextGenerationSpec in _generationSpec.ContextGenerationSpecList) { _currentContext = contextGenerationSpec; - - bool generateGetConverterMethodForTypes = false; - bool generateGetConverterMethodForProperties = false; + _generateGetConverterMethodForTypes = false; + _generateGetConverterMethodForProperties = false; foreach (TypeGenerationSpec typeGenerationSpec in _currentContext.RootSerializableTypes) { GenerateTypeInfo(typeGenerationSpec); - - generateGetConverterMethodForTypes |= typeGenerationSpec.HasTypeFactoryConverter; - generateGetConverterMethodForProperties |= typeGenerationSpec.HasPropertyFactoryConverters; } string contextName = _currentContext.ContextType.Name; @@ -126,7 +125,7 @@ public void Emit() // Add root context implementation. AddSource( $"{contextName}.g.cs", - GetRootJsonContextImplementation(generateGetConverterMethodForTypes, generateGetConverterMethodForProperties), + GetRootJsonContextImplementation(), isRootContextDef: true); // Add GetJsonTypeInfo override implementation. @@ -302,6 +301,9 @@ private void GenerateTypeInfo(TypeGenerationSpec typeGenerationSpec) Location location = typeGenerationSpec.AttributeLocation ?? _currentContext.Location; _sourceGenerationContext.ReportDiagnostic(Diagnostic.Create(DuplicateTypeName, location, new string[] { typeGenerationSpec.TypeInfoPropertyName })); } + + _generateGetConverterMethodForTypes |= typeGenerationSpec.HasTypeFactoryConverter; + _generateGetConverterMethodForProperties |= typeGenerationSpec.HasPropertyFactoryConverters; } private string GenerateForTypeWithKnownConverter(TypeGenerationSpec typeMetadata) @@ -1145,9 +1147,7 @@ private string WrapWithCheckForCustomConverter(string source, string typeCompila {IndentSource(source, numIndentations: 1)} }}"; - private string GetRootJsonContextImplementation( - bool generateGetConverterMethodForTypes, - bool generateGetConverterMethodForProperties) + private string GetRootJsonContextImplementation() { string contextTypeRef = _currentContext.ContextTypeRef; string contextTypeName = _currentContext.ContextType.Name; @@ -1171,12 +1171,12 @@ private string GetRootJsonContextImplementation( {GetFetchLogicForRuntimeSpecifiedCustomConverter()}"); - if (generateGetConverterMethodForProperties) + if (_generateGetConverterMethodForProperties) { sb.Append(GetFetchLogicForGetCustomConverter_PropertiesWithFactories()); } - if (generateGetConverterMethodForProperties || generateGetConverterMethodForTypes) + if (_generateGetConverterMethodForProperties || _generateGetConverterMethodForTypes) { sb.Append(GetFetchLogicForGetCustomConverter_TypesWithFactories()); } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/ContextClasses.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/ContextClasses.cs index 24559eb6c86a54..d7a76e4817b668 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/ContextClasses.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/ContextClasses.cs @@ -42,8 +42,8 @@ public interface ITestContext public JsonTypeInfo StructWithCustomConverterFactory { get; } public JsonTypeInfo ClassWithCustomConverterProperty { get; } public JsonTypeInfo StructWithCustomConverterProperty { get; } - public JsonTypeInfo ClassWithCustomConverterPropertyFactory { get; } - public JsonTypeInfo StructWithCustomConverterPropertyFactory { get; } + public JsonTypeInfo ClassWithCustomConverterFactoryProperty { get; } + public JsonTypeInfo StructWithCustomConverterFactoryProperty { get; } public JsonTypeInfo ClassWithBadCustomConverter { get; } public JsonTypeInfo StructWithBadCustomConverter { get; } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/JsonSerializerContextTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/JsonSerializerContextTests.cs index 49deae9acadd78..7bc91507d9ad53 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/JsonSerializerContextTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/JsonSerializerContextTests.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections.Generic; using System.Reflection; using System.Text.Json.Serialization; using Microsoft.DotNet.RemoteExecutor; @@ -89,5 +90,44 @@ internal record Person(string FirstName, string LastName); internal partial class PersonJsonContext : JsonSerializerContext { } + + // Regression test for https://github.com/dotnet/runtime/issues/62079 + [Fact] + public static void SupportsPropertiesWithCustomConverterFactory() + { + var value = new ClassWithCustomConverterFactoryProperty { MyEnum = Serialization.Tests.SampleEnum.MinZero }; + string json = JsonSerializer.Serialize(value, SingleClassWithCustomConverterFactoryPropertyContext.Default.ClassWithCustomConverterFactoryProperty); + Assert.Equal(@"{""MyEnum"":""MinZero""}", json); + } + + public class ParentClass + { + public ClassWithCustomConverterFactoryProperty? Child { get; set; } + } + + [JsonSerializable(typeof(ParentClass))] + internal partial class SingleClassWithCustomConverterFactoryPropertyContext : JsonSerializerContext + { + } + + // Regression test for https://github.com/dotnet/runtime/issues/61860 + [Fact] + public static void SupportsGenericParameterWithCustomConverterFactory() + { + var value = new List { TestEnum.Cee }; + string json = JsonSerializer.Serialize(value, GenericParameterWithCustomConverterFactoryContext.Default.ListTestEnum); + Assert.Equal(@"[""Cee""]", json); + } + + [JsonConverter(typeof(JsonStringEnumConverter))] + public enum TestEnum + { + Aye, Bee, Cee + } + + [JsonSerializable(typeof(List))] + internal partial class GenericParameterWithCustomConverterFactoryContext : JsonSerializerContext + { + } } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/MetadataAndSerializationContextTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/MetadataAndSerializationContextTests.cs index 81513b46499a70..2c7d518e87a7bd 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/MetadataAndSerializationContextTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/MetadataAndSerializationContextTests.cs @@ -36,8 +36,8 @@ namespace System.Text.Json.SourceGeneration.Tests [JsonSerializable(typeof(StructWithCustomConverterFactory))] [JsonSerializable(typeof(ClassWithCustomConverterProperty))] [JsonSerializable(typeof(StructWithCustomConverterProperty))] - [JsonSerializable(typeof(ClassWithCustomConverterPropertyFactory))] - [JsonSerializable(typeof(StructWithCustomConverterPropertyFactory))] + [JsonSerializable(typeof(ClassWithCustomConverterFactoryProperty))] + [JsonSerializable(typeof(StructWithCustomConverterFactoryProperty))] [JsonSerializable(typeof(ClassWithBadCustomConverter))] [JsonSerializable(typeof(StructWithBadCustomConverter))] internal partial class MetadataAndSerializationContext : JsonSerializerContext, ITestContext @@ -81,8 +81,8 @@ public override void EnsureFastPathGeneratedAsExpected() Assert.NotNull(MetadataAndSerializationContext.Default.StructWithCustomConverterFactory); Assert.NotNull(MetadataAndSerializationContext.Default.ClassWithCustomConverterProperty); Assert.NotNull(MetadataAndSerializationContext.Default.StructWithCustomConverterProperty); - Assert.NotNull(MetadataAndSerializationContext.Default.ClassWithCustomConverterPropertyFactory); - Assert.NotNull(MetadataAndSerializationContext.Default.StructWithCustomConverterPropertyFactory); + Assert.NotNull(MetadataAndSerializationContext.Default.ClassWithCustomConverterFactoryProperty); + Assert.NotNull(MetadataAndSerializationContext.Default.StructWithCustomConverterFactoryProperty); Assert.Throws(() => MetadataAndSerializationContext.Default.ClassWithBadCustomConverter); Assert.Throws(() => MetadataAndSerializationContext.Default.StructWithBadCustomConverter); } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/MetadataContextTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/MetadataContextTests.cs index f83a2fc923421b..4d3323cb7f9b3c 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/MetadataContextTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/MetadataContextTests.cs @@ -35,8 +35,8 @@ namespace System.Text.Json.SourceGeneration.Tests [JsonSerializable(typeof(StructWithCustomConverterFactory), GenerationMode = JsonSourceGenerationMode.Metadata)] [JsonSerializable(typeof(ClassWithCustomConverterProperty), GenerationMode = JsonSourceGenerationMode.Metadata)] [JsonSerializable(typeof(StructWithCustomConverterProperty), GenerationMode = JsonSourceGenerationMode.Metadata)] - [JsonSerializable(typeof(ClassWithCustomConverterPropertyFactory), GenerationMode = JsonSourceGenerationMode.Metadata)] - [JsonSerializable(typeof(StructWithCustomConverterPropertyFactory), GenerationMode = JsonSourceGenerationMode.Metadata)] + [JsonSerializable(typeof(ClassWithCustomConverterFactoryProperty), GenerationMode = JsonSourceGenerationMode.Metadata)] + [JsonSerializable(typeof(StructWithCustomConverterFactoryProperty), GenerationMode = JsonSourceGenerationMode.Metadata)] [JsonSerializable(typeof(ClassWithBadCustomConverter), GenerationMode = JsonSourceGenerationMode.Metadata)] [JsonSerializable(typeof(StructWithBadCustomConverter), GenerationMode = JsonSourceGenerationMode.Metadata)] internal partial class MetadataWithPerTypeAttributeContext : JsonSerializerContext, ITestContext @@ -79,8 +79,8 @@ public override void EnsureFastPathGeneratedAsExpected() Assert.Null(MetadataWithPerTypeAttributeContext.Default.StructWithCustomConverterFactory.SerializeHandler); Assert.Null(MetadataWithPerTypeAttributeContext.Default.ClassWithCustomConverterProperty.SerializeHandler); Assert.Null(MetadataWithPerTypeAttributeContext.Default.StructWithCustomConverterProperty.SerializeHandler); - Assert.Null(MetadataWithPerTypeAttributeContext.Default.ClassWithCustomConverterPropertyFactory.SerializeHandler); - Assert.Null(MetadataWithPerTypeAttributeContext.Default.StructWithCustomConverterPropertyFactory.SerializeHandler); + Assert.Null(MetadataWithPerTypeAttributeContext.Default.ClassWithCustomConverterFactoryProperty.SerializeHandler); + Assert.Null(MetadataWithPerTypeAttributeContext.Default.StructWithCustomConverterFactoryProperty.SerializeHandler); Assert.Throws(() => MetadataWithPerTypeAttributeContext.Default.ClassWithBadCustomConverter.SerializeHandler); Assert.Throws(() => MetadataWithPerTypeAttributeContext.Default.StructWithBadCustomConverter.SerializeHandler); } @@ -116,8 +116,8 @@ public override void EnsureFastPathGeneratedAsExpected() [JsonSerializable(typeof(StructWithCustomConverterFactory))] [JsonSerializable(typeof(ClassWithCustomConverterProperty))] [JsonSerializable(typeof(StructWithCustomConverterProperty))] - [JsonSerializable(typeof(ClassWithCustomConverterPropertyFactory))] - [JsonSerializable(typeof(StructWithCustomConverterPropertyFactory))] + [JsonSerializable(typeof(ClassWithCustomConverterFactoryProperty))] + [JsonSerializable(typeof(StructWithCustomConverterFactoryProperty))] [JsonSerializable(typeof(ClassWithBadCustomConverter))] [JsonSerializable(typeof(StructWithBadCustomConverter))] internal partial class MetadataContext : JsonSerializerContext, ITestContext @@ -183,8 +183,8 @@ public override void EnsureFastPathGeneratedAsExpected() Assert.Null(MetadataContext.Default.StructWithCustomConverterFactory.SerializeHandler); Assert.Null(MetadataContext.Default.ClassWithCustomConverterProperty.SerializeHandler); Assert.Null(MetadataContext.Default.StructWithCustomConverterProperty.SerializeHandler); - Assert.Null(MetadataContext.Default.ClassWithCustomConverterPropertyFactory.SerializeHandler); - Assert.Null(MetadataContext.Default.StructWithCustomConverterPropertyFactory.SerializeHandler); + Assert.Null(MetadataContext.Default.ClassWithCustomConverterFactoryProperty.SerializeHandler); + Assert.Null(MetadataContext.Default.StructWithCustomConverterFactoryProperty.SerializeHandler); Assert.Throws(() => MetadataContext.Default.ClassWithBadCustomConverter.SerializeHandler); Assert.Throws(() => MetadataContext.Default.StructWithBadCustomConverter.SerializeHandler); } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/MixedModeContextTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/MixedModeContextTests.cs index 060db030bcadab..afdb3f93184193 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/MixedModeContextTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/MixedModeContextTests.cs @@ -36,8 +36,8 @@ namespace System.Text.Json.SourceGeneration.Tests [JsonSerializable(typeof(StructWithCustomConverterFactory), GenerationMode = JsonSourceGenerationMode.Metadata | JsonSourceGenerationMode.Serialization)] [JsonSerializable(typeof(ClassWithCustomConverterProperty), GenerationMode = JsonSourceGenerationMode.Metadata | JsonSourceGenerationMode.Serialization)] [JsonSerializable(typeof(StructWithCustomConverterProperty), GenerationMode = JsonSourceGenerationMode.Metadata | JsonSourceGenerationMode.Serialization)] - [JsonSerializable(typeof(ClassWithCustomConverterPropertyFactory), GenerationMode = JsonSourceGenerationMode.Metadata | JsonSourceGenerationMode.Serialization)] - [JsonSerializable(typeof(StructWithCustomConverterPropertyFactory), GenerationMode = JsonSourceGenerationMode.Metadata | JsonSourceGenerationMode.Serialization)] + [JsonSerializable(typeof(ClassWithCustomConverterFactoryProperty), GenerationMode = JsonSourceGenerationMode.Metadata | JsonSourceGenerationMode.Serialization)] + [JsonSerializable(typeof(StructWithCustomConverterFactoryProperty), GenerationMode = JsonSourceGenerationMode.Metadata | JsonSourceGenerationMode.Serialization)] [JsonSerializable(typeof(ClassWithBadCustomConverter), GenerationMode = JsonSourceGenerationMode.Metadata | JsonSourceGenerationMode.Serialization)] [JsonSerializable(typeof(StructWithBadCustomConverter), GenerationMode = JsonSourceGenerationMode.Metadata | JsonSourceGenerationMode.Serialization)] internal partial class MixedModeContext : JsonSerializerContext, ITestContext @@ -81,8 +81,8 @@ public override void EnsureFastPathGeneratedAsExpected() Assert.Null(MixedModeContext.Default.StructWithCustomConverterFactory.SerializeHandler); Assert.Null(MixedModeContext.Default.ClassWithCustomConverterProperty.SerializeHandler); Assert.Null(MixedModeContext.Default.StructWithCustomConverterProperty.SerializeHandler); - Assert.Null(MixedModeContext.Default.ClassWithCustomConverterPropertyFactory.SerializeHandler); - Assert.Null(MixedModeContext.Default.StructWithCustomConverterPropertyFactory.SerializeHandler); + Assert.Null(MixedModeContext.Default.ClassWithCustomConverterFactoryProperty.SerializeHandler); + Assert.Null(MixedModeContext.Default.StructWithCustomConverterFactoryProperty.SerializeHandler); Assert.Throws(() => MixedModeContext.Default.ClassWithBadCustomConverter.SerializeHandler); Assert.Throws(() => MixedModeContext.Default.StructWithBadCustomConverter.SerializeHandler); } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/RealWorldContextTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/RealWorldContextTests.cs index ebfd93c01e3717..1e2a9e5fe6be45 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/RealWorldContextTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/RealWorldContextTests.cs @@ -246,28 +246,28 @@ public virtual void RoundTripWithCustomPropertyConverterFactory_Class() { const string Json = "{\"MyEnum\":\"One\"}"; - ClassWithCustomConverterPropertyFactory obj = new() + ClassWithCustomConverterFactoryProperty obj = new() { MyEnum = SampleEnum.One }; if (DefaultContext.JsonSourceGenerationMode == JsonSourceGenerationMode.Serialization) { - Assert.Throws(() => JsonSerializer.Serialize(obj, DefaultContext.ClassWithCustomConverterPropertyFactory)); + Assert.Throws(() => JsonSerializer.Serialize(obj, DefaultContext.ClassWithCustomConverterFactoryProperty)); } else { - string json = JsonSerializer.Serialize(obj, DefaultContext.ClassWithCustomConverterPropertyFactory); + string json = JsonSerializer.Serialize(obj, DefaultContext.ClassWithCustomConverterFactoryProperty); Assert.Equal(Json, json); } if (DefaultContext.JsonSourceGenerationMode == JsonSourceGenerationMode.Serialization) { - Assert.Throws(() => JsonSerializer.Serialize(obj, DefaultContext.ClassWithCustomConverterPropertyFactory)); + Assert.Throws(() => JsonSerializer.Serialize(obj, DefaultContext.ClassWithCustomConverterFactoryProperty)); } else { - obj = JsonSerializer.Deserialize(Json, DefaultContext.ClassWithCustomConverterPropertyFactory); + obj = JsonSerializer.Deserialize(Json, DefaultContext.ClassWithCustomConverterFactoryProperty); Assert.Equal(SampleEnum.One, obj.MyEnum); } } @@ -277,28 +277,28 @@ public virtual void RoundTripWithCustomPropertyConverterFactory_Struct() { const string Json = "{\"MyEnum\":\"One\"}"; - StructWithCustomConverterPropertyFactory obj = new() + StructWithCustomConverterFactoryProperty obj = new() { MyEnum = SampleEnum.One }; if (DefaultContext.JsonSourceGenerationMode == JsonSourceGenerationMode.Serialization) { - Assert.Throws(() => JsonSerializer.Serialize(obj, DefaultContext.StructWithCustomConverterPropertyFactory)); + Assert.Throws(() => JsonSerializer.Serialize(obj, DefaultContext.StructWithCustomConverterFactoryProperty)); } else { - string json = JsonSerializer.Serialize(obj, DefaultContext.StructWithCustomConverterPropertyFactory); + string json = JsonSerializer.Serialize(obj, DefaultContext.StructWithCustomConverterFactoryProperty); Assert.Equal(Json, json); } if (DefaultContext.JsonSourceGenerationMode == JsonSourceGenerationMode.Serialization) { - Assert.Throws(() => JsonSerializer.Serialize(obj, DefaultContext.StructWithCustomConverterPropertyFactory)); + Assert.Throws(() => JsonSerializer.Serialize(obj, DefaultContext.StructWithCustomConverterFactoryProperty)); } else { - obj = JsonSerializer.Deserialize(Json, DefaultContext.StructWithCustomConverterPropertyFactory); + obj = JsonSerializer.Deserialize(Json, DefaultContext.StructWithCustomConverterFactoryProperty); Assert.Equal(SampleEnum.One, obj.MyEnum); } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/SerializationContextTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/SerializationContextTests.cs index 57f663e8b36339..c658661b7a8c3f 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/SerializationContextTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/SerializationContextTests.cs @@ -36,8 +36,8 @@ namespace System.Text.Json.SourceGeneration.Tests [JsonSerializable(typeof(StructWithCustomConverterFactory))] [JsonSerializable(typeof(ClassWithCustomConverterProperty))] [JsonSerializable(typeof(StructWithCustomConverterProperty))] - [JsonSerializable(typeof(ClassWithCustomConverterPropertyFactory))] - [JsonSerializable(typeof(StructWithCustomConverterPropertyFactory))] + [JsonSerializable(typeof(ClassWithCustomConverterFactoryProperty))] + [JsonSerializable(typeof(StructWithCustomConverterFactoryProperty))] [JsonSerializable(typeof(ClassWithBadCustomConverter))] [JsonSerializable(typeof(StructWithBadCustomConverter))] internal partial class SerializationContext : JsonSerializerContext, ITestContext @@ -74,8 +74,8 @@ internal partial class SerializationContext : JsonSerializerContext, ITestContex [JsonSerializable(typeof(StructWithCustomConverterFactory), GenerationMode = JsonSourceGenerationMode.Serialization)] [JsonSerializable(typeof(ClassWithCustomConverterProperty), GenerationMode = JsonSourceGenerationMode.Serialization)] [JsonSerializable(typeof(StructWithCustomConverterProperty), GenerationMode = JsonSourceGenerationMode.Serialization)] - [JsonSerializable(typeof(ClassWithCustomConverterPropertyFactory), GenerationMode = JsonSourceGenerationMode.Serialization)] - [JsonSerializable(typeof(StructWithCustomConverterPropertyFactory), GenerationMode = JsonSourceGenerationMode.Serialization)] + [JsonSerializable(typeof(ClassWithCustomConverterFactoryProperty), GenerationMode = JsonSourceGenerationMode.Serialization)] + [JsonSerializable(typeof(StructWithCustomConverterFactoryProperty), GenerationMode = JsonSourceGenerationMode.Serialization)] [JsonSerializable(typeof(ClassWithBadCustomConverter), GenerationMode = JsonSourceGenerationMode.Serialization)] [JsonSerializable(typeof(StructWithBadCustomConverter), GenerationMode = JsonSourceGenerationMode.Serialization)] internal partial class SerializationWithPerTypeAttributeContext : JsonSerializerContext, ITestContext @@ -113,8 +113,8 @@ internal partial class SerializationWithPerTypeAttributeContext : JsonSerializer [JsonSerializable(typeof(StructWithCustomConverterFactory), GenerationMode = JsonSourceGenerationMode.Serialization)] [JsonSerializable(typeof(ClassWithCustomConverterProperty), GenerationMode = JsonSourceGenerationMode.Serialization)] [JsonSerializable(typeof(StructWithCustomConverterProperty), GenerationMode = JsonSourceGenerationMode.Serialization)] - [JsonSerializable(typeof(ClassWithCustomConverterPropertyFactory), GenerationMode = JsonSourceGenerationMode.Serialization)] - [JsonSerializable(typeof(StructWithCustomConverterPropertyFactory), GenerationMode = JsonSourceGenerationMode.Serialization)] + [JsonSerializable(typeof(ClassWithCustomConverterFactoryProperty), GenerationMode = JsonSourceGenerationMode.Serialization)] + [JsonSerializable(typeof(StructWithCustomConverterFactoryProperty), GenerationMode = JsonSourceGenerationMode.Serialization)] [JsonSerializable(typeof(ClassWithBadCustomConverter), GenerationMode = JsonSourceGenerationMode.Serialization)] [JsonSerializable(typeof(StructWithBadCustomConverter), GenerationMode = JsonSourceGenerationMode.Serialization)] internal partial class SerializationContextWithCamelCase : JsonSerializerContext, ITestContext @@ -466,8 +466,8 @@ public override void EnsureFastPathGeneratedAsExpected() Assert.Null(SerializationWithPerTypeAttributeContext.Default.StructWithCustomConverterFactory.SerializeHandler); Assert.Null(SerializationWithPerTypeAttributeContext.Default.ClassWithCustomConverterProperty.SerializeHandler); Assert.Null(SerializationWithPerTypeAttributeContext.Default.StructWithCustomConverterProperty.SerializeHandler); - Assert.Null(SerializationWithPerTypeAttributeContext.Default.ClassWithCustomConverterPropertyFactory.SerializeHandler); - Assert.Null(SerializationWithPerTypeAttributeContext.Default.StructWithCustomConverterPropertyFactory.SerializeHandler); + Assert.Null(SerializationWithPerTypeAttributeContext.Default.ClassWithCustomConverterFactoryProperty.SerializeHandler); + Assert.Null(SerializationWithPerTypeAttributeContext.Default.StructWithCustomConverterFactoryProperty.SerializeHandler); Assert.Throws(() => SerializationWithPerTypeAttributeContext.Default.ClassWithBadCustomConverter.SerializeHandler); Assert.Throws(() => SerializationWithPerTypeAttributeContext.Default.StructWithBadCustomConverter.SerializeHandler); } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/TestClasses.CustomConverters.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/TestClasses.CustomConverters.cs index f2be0926a0bc89..1fff23f8a6c55c 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/TestClasses.CustomConverters.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Tests/TestClasses.CustomConverters.cs @@ -245,13 +245,13 @@ public struct StructWithCustomConverterProperty public ClassWithCustomConverterProperty.NestedPoco Property { get; set; } } - public struct ClassWithCustomConverterPropertyFactory + public class ClassWithCustomConverterFactoryProperty { [JsonConverter(typeof(JsonStringEnumConverter))] // This converter is a JsonConverterFactory public Serialization.Tests.SampleEnum MyEnum { get; set; } } - public struct StructWithCustomConverterPropertyFactory + public struct StructWithCustomConverterFactoryProperty { [JsonConverter(typeof(JsonStringEnumConverter))] // This converter is a JsonConverterFactory public Serialization.Tests.SampleEnum MyEnum { get; set; } From beedfae9073b929f9a06d34208c49a552440dd12 Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Wed, 12 Jan 2022 16:24:11 -0800 Subject: [PATCH 36/39] [release/6.0] Build all packages when in source-build (#63653) --- eng/packaging.targets | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/eng/packaging.targets b/eng/packaging.targets index 6bcf86dc9f2e39..ee8e95167d9f88 100644 --- a/eng/packaging.targets +++ b/eng/packaging.targets @@ -34,6 +34,10 @@ '$(IsRIDSpecificProject)' != 'true' and '$(PreReleaseVersionLabel)' == 'servicing' and '$(GitHubRepositoryName)' != 'runtimelab'">false + + true $(XmlDocFileRoot)1033\$(AssemblyName).xml true @@ -279,7 +283,7 @@ From fe7335c989a4356b79b77e62c8a8ad4c901ff5e5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 13 Jan 2022 08:47:36 -0800 Subject: [PATCH 37/39] [release/6.0] Use "read" to fetch misaligned 64bit values from bundle header (#63519) * Use "read" to fetch misaligned 64bit values * remove read_direct * rename read() -> read_byte() Co-authored-by: vsadov <8218165+VSadov@users.noreply.github.com> --- src/native/corehost/bundle/file_entry.cpp | 15 +++++++++------ src/native/corehost/bundle/header.cpp | 12 ++++++------ src/native/corehost/bundle/reader.cpp | 4 ++-- src/native/corehost/bundle/reader.h | 12 +----------- 4 files changed, 18 insertions(+), 25 deletions(-) diff --git a/src/native/corehost/bundle/file_entry.cpp b/src/native/corehost/bundle/file_entry.cpp index ace0ef514927e6..a6610d7de97b92 100644 --- a/src/native/corehost/bundle/file_entry.cpp +++ b/src/native/corehost/bundle/file_entry.cpp @@ -19,15 +19,18 @@ file_entry_t file_entry_t::read(reader_t &reader, uint32_t bundle_major_version, // First read the fixed-sized portion of file-entry file_entry_fixed_t fixed_data; - fixed_data.offset = *(int64_t*)reader.read_direct(sizeof(int64_t)); - fixed_data.size = *(int64_t*)reader.read_direct(sizeof(int64_t)); + // NB: the file data is potentially unaligned, thus we use "read" to fetch 64bit values + reader.read(&fixed_data.offset, sizeof(int64_t)); + reader.read(&fixed_data.size, sizeof(int64_t)); // compressedSize is present only in v6+ headers - fixed_data.compressedSize = bundle_major_version >= 6 ? - *(int64_t*)reader.read_direct(sizeof(int64_t)) : - 0; + fixed_data.compressedSize = 0; + if (bundle_major_version >= 6) + { + reader.read(&fixed_data.compressedSize, sizeof(int64_t)); + } - fixed_data.type = *(file_type_t*)reader.read_direct(sizeof(file_type_t)); + fixed_data.type = (file_type_t)reader.read_byte(); file_entry_t entry(&fixed_data, force_extraction); diff --git a/src/native/corehost/bundle/header.cpp b/src/native/corehost/bundle/header.cpp index 0eb42d94f72182..bc549f4c7af8c7 100644 --- a/src/native/corehost/bundle/header.cpp +++ b/src/native/corehost/bundle/header.cpp @@ -24,23 +24,23 @@ bool header_fixed_t::is_valid() const header_t header_t::read(reader_t& reader) { - const header_fixed_t* fixed_header = reinterpret_cast(reader.read_direct(sizeof(header_fixed_t))); + header_fixed_t fixed_header; + reader.read(&fixed_header, sizeof(header_fixed_t)); - if (!fixed_header->is_valid()) + if (!fixed_header.is_valid()) { trace::error(_X("Failure processing application bundle.")); - trace::error(_X("Bundle header version compatibility check failed. Header version: %d.%d"), fixed_header->major_version, fixed_header->minor_version); + trace::error(_X("Bundle header version compatibility check failed. Header version: %d.%d"), fixed_header.major_version, fixed_header.minor_version); throw StatusCode::BundleExtractionFailure; } - header_t header(fixed_header->major_version, fixed_header->minor_version, fixed_header->num_embedded_files); + header_t header(fixed_header.major_version, fixed_header.minor_version, fixed_header.num_embedded_files); // bundle_id is a component of the extraction path reader.read_path_string(header.m_bundle_id); - const header_fixed_v2_t *v2_header = reinterpret_cast(reader.read_direct(sizeof(header_fixed_v2_t))); - header.m_v2_header = *v2_header; + reader.read(&header.m_v2_header, sizeof(header_fixed_v2_t)); return header; } diff --git a/src/native/corehost/bundle/reader.cpp b/src/native/corehost/bundle/reader.cpp index a7b5e580f0b562..8461e5db920be2 100644 --- a/src/native/corehost/bundle/reader.cpp +++ b/src/native/corehost/bundle/reader.cpp @@ -54,7 +54,7 @@ size_t reader_t::read_path_length() { size_t length = 0; - int8_t first_byte = read(); + int8_t first_byte = read_byte(); // If the high bit is set, it means there are more bytes to read. if ((first_byte & 0x80) == 0) @@ -63,7 +63,7 @@ size_t reader_t::read_path_length() } else { - int8_t second_byte = read(); + int8_t second_byte = read_byte(); if (second_byte & 0x80) { diff --git a/src/native/corehost/bundle/reader.h b/src/native/corehost/bundle/reader.h index 576ecd04c8e0fe..a4c1e07be19ece 100644 --- a/src/native/corehost/bundle/reader.h +++ b/src/native/corehost/bundle/reader.h @@ -81,7 +81,7 @@ namespace bundle return m_ptr; } - int8_t read() + int8_t read_byte() { bounds_check(); return *m_ptr++; @@ -100,16 +100,6 @@ namespace bundle m_ptr += len; } - // Return a pointer to the requested bytes within the memory-mapped file. - // Skip over len bytes. - const char* read_direct(int64_t len) - { - bounds_check(len); - const char *ptr = m_ptr; - m_ptr += len; - return ptr; - } - size_t read_path_length(); size_t read_path_string(pal::string_t &str); From 3c2d2a14d466c91304ec5f616e29d0236ddb06dd Mon Sep 17 00:00:00 2001 From: Mike McLaughlin Date: Thu, 13 Jan 2022 10:20:13 -0800 Subject: [PATCH 38/39] Fix the MacOS remote unwinder for VS4Mac (#63405) * Fix the MacOS remote unwinder for VS4Mac The wrong module was being passed to the remote unwinder because the load bias for shared modules was being calculated incorrectly. Issue: https://github.com/dotnet/runtime/issues/63309 * Fix native frame unwind in syscall on arm64 for VS4Mac crash report From PR in main: https://github.com/dotnet/runtime/pull/63598 Add arm64 version of StepWithCompactNoEncoding for syscall leaf node wrappers that have compact encoding of 0. Fix ReadCompactEncodingRegister so it actually decrements the addr. Change StepWithCompactEncodingArm64 to match what MacOS libunwind does for framed and frameless stepping. arm64 can have frames with the same SP (but different IPs). Increment SP for this condition so createdump's unwind loop doesn't break out on the "SP not increasing" check and the frames are added to the thread frame list in the correct order. Add getting the unwind info for tail called functions like this: __ZL14PROCEndProcessPvji: 36630: f6 57 bd a9 stp x22, x21, [sp, #-48]! 36634: f4 4f 01 a9 stp x20, x19, [sp, #16] 36638: fd 7b 02 a9 stp x29, x30, [sp, #32] 3663c: fd 83 00 91 add x29, sp, #32 ... 367ac: e9 01 80 52 mov w9, #15 367b0: 7f 3e 02 71 cmp w19, #143 367b4: 20 01 88 1a csel w0, w9, w8, eq 367b8: 2e 00 00 94 bl _PROCAbort _TerminateProcess: -> 367bc: 22 00 80 52 mov w2, #1 367c0: 9c ff ff 17 b __ZL14PROCEndProcessPvji The IP (367bc) returns the (incorrect) frameless encoding with nothing on the stack (uses an incorrect LR to unwind). To fix this get the unwind info for PC -1 which points to PROCEndProcess with the correct unwind info. This matches how lldb unwinds this frame. Always address module segment to IP lookup list instead of checking the module regions. Strip pointer authentication bits on PC/LR. --- src/coreclr/debug/createdump/crashinfomac.cpp | 6 +- src/coreclr/debug/createdump/stackframe.h | 9 +- src/coreclr/debug/createdump/threadinfo.cpp | 10 ++ src/coreclr/debug/dbgutil/machoreader.cpp | 5 +- .../pal/src/exception/remote-unwind.cpp | 129 +++++++++++++----- 5 files changed, 118 insertions(+), 41 deletions(-) diff --git a/src/coreclr/debug/createdump/crashinfomac.cpp b/src/coreclr/debug/createdump/crashinfomac.cpp index ad9c247e37dfdc..3ada3bd767c964 100644 --- a/src/coreclr/debug/createdump/crashinfomac.cpp +++ b/src/coreclr/debug/createdump/crashinfomac.cpp @@ -277,6 +277,9 @@ void CrashInfo::VisitSegment(MachOModule& module, const segment_command_64& segm uint64_t start = segment.vmaddr + module.LoadBias(); uint64_t end = start + segment.vmsize; + // Add this module segment to the set used by the thread unwinding to lookup the module base address for an ip. + AddModuleAddressRange(start, end, module.BaseAddress()); + // Round to page boundary start = start & PAGE_MASK; _ASSERTE(start > 0); @@ -297,9 +300,6 @@ void CrashInfo::VisitSegment(MachOModule& module, const segment_command_64& segm } // Add this module segment to the module mappings list m_moduleMappings.insert(moduleRegion); - - // Add this module segment to the set used by the thread unwinding to lookup the module base address for an ip. - AddModuleAddressRange(start, end, module.BaseAddress()); } else { diff --git a/src/coreclr/debug/createdump/stackframe.h b/src/coreclr/debug/createdump/stackframe.h index 75e20d93120c0c..00c3a1cfb7fd8f 100644 --- a/src/coreclr/debug/createdump/stackframe.h +++ b/src/coreclr/debug/createdump/stackframe.h @@ -66,9 +66,16 @@ struct StackFrame } } +// See comment in threadinfo.cpp UnwindNativeFrames function +#if defined(__aarch64__) + #define STACK_POINTER_MASK ~0x7 +#else + #define STACK_POINTER_MASK ~0x0 +#endif + inline uint64_t ModuleAddress() const { return m_moduleAddress; } inline uint64_t InstructionPointer() const { return m_instructionPointer; } - inline uint64_t StackPointer() const { return m_stackPointer; } + inline uint64_t StackPointer() const { return m_stackPointer & STACK_POINTER_MASK; } inline uint32_t NativeOffset() const { return m_nativeOffset; } inline uint32_t Token() const { return m_token; } inline uint32_t ILOffset() const { return m_ilOffset; } diff --git a/src/coreclr/debug/createdump/threadinfo.cpp b/src/coreclr/debug/createdump/threadinfo.cpp index 82509f52750653..e64eafc8d29a96 100644 --- a/src/coreclr/debug/createdump/threadinfo.cpp +++ b/src/coreclr/debug/createdump/threadinfo.cpp @@ -53,6 +53,16 @@ ThreadInfo::UnwindNativeFrames(CONTEXT* pContext) uint64_t ip = 0, sp = 0; GetFrameLocation(pContext, &ip, &sp); +#if defined(__aarch64__) + // ARM64 can have frames with the same SP but different IPs. Increment sp so it gets added to the stack + // frames in the correct order and to prevent the below loop termination on non-increasing sp. Since stack + // pointers are always 8 byte align, this increase is masked off in StackFrame::StackPointer() to get the + // original stack pointer. + if (sp == previousSp && ip != previousIp) + { + sp++; + } +#endif if (ip == 0 || sp <= previousSp) { TRACE_VERBOSE("Unwind: sp not increasing or ip == 0 sp %p ip %p\n", (void*)sp, (void*)ip); break; diff --git a/src/coreclr/debug/dbgutil/machoreader.cpp b/src/coreclr/debug/dbgutil/machoreader.cpp index d2801547cb9ba9..7fef34e1afb16d 100644 --- a/src/coreclr/debug/dbgutil/machoreader.cpp +++ b/src/coreclr/debug/dbgutil/machoreader.cpp @@ -229,9 +229,8 @@ MachOModule::ReadLoadCommands() m_segments.push_back(segment); // Calculate the load bias for the module. This is the value to add to the vmaddr of a - // segment to get the actual address. For shared modules, this is 0 since those segments - // are absolute address. - if (segment->fileoff == 0 && segment->filesize > 0) + // segment to get the actual address. + if (strcmp(segment->segname, SEG_TEXT) == 0) { m_loadBias = m_baseAddress - segment->vmaddr; } diff --git a/src/coreclr/pal/src/exception/remote-unwind.cpp b/src/coreclr/pal/src/exception/remote-unwind.cpp index af0293ba5bc60c..b27fc9680bbedf 100644 --- a/src/coreclr/pal/src/exception/remote-unwind.cpp +++ b/src/coreclr/pal/src/exception/remote-unwind.cpp @@ -57,6 +57,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #include #include #include "compact_unwind_encoding.h" +#define MACOS_ARM64_POINTER_AUTH_MASK 0x7fffffffffffull #endif // Sub-headers included from the libunwind.h contain an empty struct @@ -1422,25 +1423,56 @@ StepWithCompactNoEncoding(const libunwindInfo* info) #if defined(TARGET_ARM64) -inline static bool +#define ARM64_SYSCALL_OPCODE 0xD4001001 +#define ARM64_BL_OPCODE_MASK 0xFC000000 +#define ARM64_BL_OPCODE 0x94000000 +#define ARM64_BLR_OPCODE_MASK 0xFFFFFC00 +#define ARM64_BLR_OPCODE 0xD63F0000 +#define ARM64_BLRA_OPCODE_MASK 0xFEFFF800 +#define ARM64_BLRA_OPCODE 0xD63F0800 + +static bool +StepWithCompactNoEncoding(const libunwindInfo* info) +{ + // Check that the function is a syscall "wrapper" and assume there is no frame and pop the return address. + uint32_t opcode; + unw_word_t addr = info->Context->Pc - sizeof(opcode); + if (!ReadValue32(info, &addr, &opcode)) { + ERROR("StepWithCompactNoEncoding: can read opcode %p\n", (void*)addr); + return false; + } + // Is the IP pointing just after a "syscall" opcode? + if (opcode != ARM64_SYSCALL_OPCODE) { + ERROR("StepWithCompactNoEncoding: not in syscall wrapper function\n"); + return false; + } + // Pop the return address from the stack + info->Context->Pc = info->Context->Lr; + TRACE("StepWithCompactNoEncoding: SUCCESS new pc %p sp %p\n", (void*)info->Context->Pc, (void*)info->Context->Sp); + return true; +} + +static bool ReadCompactEncodingRegister(const libunwindInfo* info, unw_word_t* addr, DWORD64* reg) { - *addr -= sizeof(uint64_t); - if (!ReadValue64(info, addr, (uint64_t*)reg)) { + uint64_t value; + if (!info->ReadMemory((PVOID)*addr, &value, sizeof(value))) { return false; } + *reg = VAL64(value); + *addr -= sizeof(uint64_t); return true; } -inline static bool -ReadCompactEncodingRegisterPair(const libunwindInfo* info, unw_word_t* addr, DWORD64*second, DWORD64* first) +static bool +ReadCompactEncodingRegisterPair(const libunwindInfo* info, unw_word_t* addr, DWORD64* first, DWORD64* second) { // Registers are effectively pushed in pairs // + // *first = **addr // *addr -= 8 - // **addr = *first + // *second= **addr // *addr -= 8 - // **addr = *second if (!ReadCompactEncodingRegister(info, addr, first)) { return false; } @@ -1450,8 +1482,8 @@ ReadCompactEncodingRegisterPair(const libunwindInfo* info, unw_word_t* addr, DWO return true; } -inline static bool -ReadCompactEncodingRegisterPair(const libunwindInfo* info, unw_word_t* addr, NEON128*second, NEON128* first) +static bool +ReadCompactEncodingRegisterPair(const libunwindInfo* info, unw_word_t* addr, NEON128* first, NEON128* second) { if (!ReadCompactEncodingRegisterPair(info, addr, &first->Low, &second->Low)) { return false; @@ -1484,30 +1516,28 @@ static bool StepWithCompactEncodingArm64(const libunwindInfo* info, compact_unwind_encoding_t compactEncoding, bool hasFrame) { CONTEXT* context = info->Context; + unw_word_t addr; - unw_word_t callerSp; - - if (hasFrame) { - // caller Sp is callee Fp plus saved FP and LR - callerSp = context->Fp + 2 * sizeof(uint64_t); - } else { + if (hasFrame) + { + context->Sp = context->Fp + 16; + addr = context->Fp + 8; + if (!ReadCompactEncodingRegisterPair(info, &addr, &context->Lr, &context->Fp)) { + return false; + } + // Strip pointer authentication bits + context->Lr &= MACOS_ARM64_POINTER_AUTH_MASK; + } + else + { // Get the leat significant bit in UNWIND_ARM64_FRAMELESS_STACK_SIZE_MASK uint64_t stackSizeScale = UNWIND_ARM64_FRAMELESS_STACK_SIZE_MASK & ~(UNWIND_ARM64_FRAMELESS_STACK_SIZE_MASK - 1); - uint64_t stackSize = (compactEncoding & UNWIND_ARM64_FRAMELESS_STACK_SIZE_MASK) / stackSizeScale * 16; + uint64_t stackSize = ((compactEncoding & UNWIND_ARM64_FRAMELESS_STACK_SIZE_MASK) / stackSizeScale) * 16; - callerSp = context->Sp + stackSize; + addr = context->Sp + stackSize; } - context->Sp = callerSp; - - unw_word_t addr = callerSp; - - if (hasFrame && - !ReadCompactEncodingRegisterPair(info, &addr, &context->Lr, &context->Fp)) { - return false; - } - - // unwound return address is stored in Lr + // Unwound return address is stored in Lr context->Pc = context->Lr; if (compactEncoding & UNWIND_ARM64_FRAME_X19_X20_PAIR && @@ -1546,7 +1576,10 @@ StepWithCompactEncodingArm64(const libunwindInfo* info, compact_unwind_encoding_ !ReadCompactEncodingRegisterPair(info, &addr, &context->V[14], &context->V[15])) { return false; } - + if (!hasFrame) + { + context->Sp = addr; + } TRACE("SUCCESS: compact step encoding %08x pc %p sp %p fp %p lr %p\n", compactEncoding, (void*)context->Pc, (void*)context->Sp, (void*)context->Fp, (void*)context->Lr); return true; @@ -1557,11 +1590,11 @@ StepWithCompactEncodingArm64(const libunwindInfo* info, compact_unwind_encoding_ static bool StepWithCompactEncoding(const libunwindInfo* info, compact_unwind_encoding_t compactEncoding, unw_word_t functionStart) { -#if defined(TARGET_AMD64) if (compactEncoding == 0) { return StepWithCompactNoEncoding(info); } +#if defined(TARGET_AMD64) switch (compactEncoding & UNWIND_X86_64_MODE_MASK) { case UNWIND_X86_64_MODE_RBP_FRAME: @@ -1575,11 +1608,6 @@ StepWithCompactEncoding(const libunwindInfo* info, compact_unwind_encoding_t com return false; } #elif defined(TARGET_ARM64) - if (compactEncoding == 0) - { - TRACE("Compact unwind missing for %p\n", (void*)info->Context->Pc); - return false; - } switch (compactEncoding & UNWIND_ARM64_MODE_MASK) { case UNWIND_ARM64_MODE_FRAME: @@ -1717,6 +1745,12 @@ static void UnwindContextToContext(unw_cursor_t *cursor, CONTEXT *winContext) unw_get_reg(cursor, UNW_AARCH64_X28, (unw_word_t *) &winContext->X28); unw_get_reg(cursor, UNW_AARCH64_X29, (unw_word_t *) &winContext->Fp); unw_get_reg(cursor, UNW_AARCH64_X30, (unw_word_t *) &winContext->Lr); +#ifdef __APPLE__ + // Strip pointer authentication bits which seem to be leaking out of libunwind + // Seems like ptrauth_strip() / __builtin_ptrauth_strip() should work, but currently + // errors with "this target does not support pointer authentication" + winContext->Pc = winContext->Pc & MACOS_ARM64_POINTER_AUTH_MASK; +#endif // __APPLE__ TRACE("sp %p pc %p lr %p fp %p\n", winContext->Sp, winContext->Pc, winContext->Lr, winContext->Fp); #elif defined(TARGET_S390X) unw_get_reg(cursor, UNW_REG_IP, (unw_word_t *) &winContext->PSWAddr); @@ -2126,6 +2160,33 @@ PAL_VirtualUnwindOutOfProc(CONTEXT *context, KNONVOLATILE_CONTEXT_POINTERS *cont #elif defined(TARGET_ARM64) TRACE("Unwind: pc %p sp %p fp %p\n", (void*)context->Pc, (void*)context->Sp, (void*)context->Fp); result = GetProcInfo(context->Pc, &procInfo, &info, &step, false); + if (result && step) + { + // If the PC is at the start of the function, the previous instruction is BL and the unwind encoding is frameless + // with nothing on stack (0x02000000), back up PC by 1 to the previous function and get the unwind info for that + // function. + if ((context->Pc == procInfo.start_ip) && + (procInfo.format & (UNWIND_ARM64_MODE_MASK | UNWIND_ARM64_FRAMELESS_STACK_SIZE_MASK)) == UNWIND_ARM64_MODE_FRAMELESS) + { + uint32_t opcode; + unw_word_t addr = context->Pc - sizeof(opcode); + if (ReadValue32(&info, &addr, &opcode)) + { + // Is the previous instruction a BL opcode? + if ((opcode & ARM64_BL_OPCODE_MASK) == ARM64_BL_OPCODE || + (opcode & ARM64_BLR_OPCODE_MASK) == ARM64_BLR_OPCODE || + (opcode & ARM64_BLRA_OPCODE_MASK) == ARM64_BLRA_OPCODE) + { + TRACE("Unwind: getting unwind info for PC - 1 opcode %08x\n", opcode); + result = GetProcInfo(context->Pc - 1, &procInfo, &info, &step, false); + } + else + { + TRACE("Unwind: not BL* opcode %08x\n", opcode); + } + } + } + } #else #error Unexpected architecture #endif From 6de5c5ba25886270bddfbd827b6cf0b7f8a178a2 Mon Sep 17 00:00:00 2001 From: Thays Grazia Date: Fri, 14 Jan 2022 16:00:23 -0300 Subject: [PATCH 39/39] [wasm][debugger] Missing test files (#63148) Adding test files that were not added on release 6.0 by mistake. --- .../debugger-test-with-full-debug-type.csproj | 5 +++++ .../test.cs | 20 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 src/mono/wasm/debugger/tests/debugger-test-with-full-debug-type/debugger-test-with-full-debug-type.csproj create mode 100644 src/mono/wasm/debugger/tests/debugger-test-with-full-debug-type/test.cs diff --git a/src/mono/wasm/debugger/tests/debugger-test-with-full-debug-type/debugger-test-with-full-debug-type.csproj b/src/mono/wasm/debugger/tests/debugger-test-with-full-debug-type/debugger-test-with-full-debug-type.csproj new file mode 100644 index 00000000000000..5eb1be9d8ea299 --- /dev/null +++ b/src/mono/wasm/debugger/tests/debugger-test-with-full-debug-type/debugger-test-with-full-debug-type.csproj @@ -0,0 +1,5 @@ + + + full + + \ No newline at end of file diff --git a/src/mono/wasm/debugger/tests/debugger-test-with-full-debug-type/test.cs b/src/mono/wasm/debugger/tests/debugger-test-with-full-debug-type/test.cs new file mode 100644 index 00000000000000..64609af24a547f --- /dev/null +++ b/src/mono/wasm/debugger/tests/debugger-test-with-full-debug-type/test.cs @@ -0,0 +1,20 @@ +using System; + +namespace DebuggerTests +{ + public class ClassToInspectWithDebugTypeFull + { + int a; + int b; + int c; + public ClassToInspectWithDebugTypeFull() + { + a = 10; + b = 20; + c = 30; + Console.WriteLine(a); + Console.WriteLine(b); + Console.WriteLine(c); + } + } +} \ No newline at end of file