From eac17ba7630cf6d68225c3275a7841d2bbe1b619 Mon Sep 17 00:00:00 2001 From: wtgodbe Date: Tue, 29 May 2018 13:39:57 -0700 Subject: [PATCH 01/18] Bump versions for 2.1.2 --- Packaging.props | 2 +- .../packageIndex.json | 201 ++++++++++++++---- src/packages.builds | 13 +- 3 files changed, 167 insertions(+), 49 deletions(-) diff --git a/Packaging.props b/Packaging.props index b54ccad190d2..861f0e5e4bef 100644 --- a/Packaging.props +++ b/Packaging.props @@ -2,7 +2,7 @@ false $(PackageVersionStamp) - rtm + servicing true false true diff --git a/pkg/Microsoft.Private.PackageBaseline/packageIndex.json b/pkg/Microsoft.Private.PackageBaseline/packageIndex.json index 30def071acd4..58db70f43f03 100644 --- a/pkg/Microsoft.Private.PackageBaseline/packageIndex.json +++ b/pkg/Microsoft.Private.PackageBaseline/packageIndex.json @@ -61,7 +61,8 @@ "4.0.1", "4.0.0", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": { @@ -106,7 +107,8 @@ }, "Microsoft.Diagnostics.Tracing.EventSource.Redist": { "StableVersions": [ - "2.0.0" + "2.0.0", + "2.0.1" ], "InboxOn": {}, "AssemblyVersionInPackageVersion": { @@ -125,7 +127,8 @@ "1.0.1", "1.0.2", "1.1.0", - "2.0.0" + "2.0.0", + "2.1.0" ], "BaselineVersion": "2.0.0", "InboxOn": {} @@ -136,7 +139,8 @@ "1.0.1", "1.0.2", "1.1.0", - "2.0.0" + "2.0.0", + "2.1.0" ], "BaselineVersion": "2.0.0", "InboxOn": {} @@ -182,7 +186,8 @@ "10.0.1", "10.0.0", "10.1.0", - "10.2.0" + "10.2.0", + "10.3.0" ], "BaselineVersion": "10.2.0", "InboxOn": { @@ -256,7 +261,8 @@ "StableVersions": [ "4.0.0", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": {}, @@ -271,7 +277,8 @@ "StableVersions": [ "4.0.0", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": {}, @@ -283,20 +290,30 @@ } }, "Microsoft.Win32.SystemEvents": { + "StableVersions": [ + "4.5.0" + ], "InboxOn": {}, "AssemblyVersionInPackageVersion": { "4.0.0.0": "4.5.0" } }, "Microsoft.Windows.Compatibility": { + "StableVersions": [ + "2.0.0" + ], "InboxOn": {} }, "Microsoft.Windows.Compatibility.Shims": { + "StableVersions": [ + "2.0.0" + ], "InboxOn": {} }, "Microsoft.XmlSerializer.Generator": { "StableVersions": [ - "1.0.0" + "1.0.0", + "2.0.0" ], "InboxOn": {}, "AssemblyVersionInPackageVersion": { @@ -453,7 +470,8 @@ "StableVersions": [ "4.0.0", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": {} @@ -577,7 +595,8 @@ "StableVersions": [ "4.0.0", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": { @@ -594,7 +613,8 @@ }, "System.CodeDom": { "StableVersions": [ - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": {}, @@ -687,7 +707,8 @@ "1.2.0", "1.1.36", "1.3.0", - "1.4.0" + "1.4.0", + "1.5.0" ], "BaselineVersion": "1.4.0", "InboxOn": { @@ -801,7 +822,8 @@ "4.0.10", "4.1.0", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": { @@ -830,6 +852,9 @@ } }, "System.ComponentModel.Composition": { + "StableVersions": [ + "4.5.0" + ], "InboxOn": { "net45": "4.0.0.0", "monoandroid10": "Any", @@ -963,7 +988,8 @@ "System.Composition": { "StableVersions": [ "1.0.31", - "1.1.0" + "1.1.0", + "1.2.0" ], "BaselineVersion": "1.1.0", "InboxOn": {} @@ -971,7 +997,8 @@ "System.Composition.AttributedModel": { "StableVersions": [ "1.0.31", - "1.1.0" + "1.1.0", + "1.2.0" ], "BaselineVersion": "1.1.0", "InboxOn": {}, @@ -984,7 +1011,8 @@ "System.Composition.Convention": { "StableVersions": [ "1.0.31", - "1.1.0" + "1.1.0", + "1.2.0" ], "BaselineVersion": "1.1.0", "InboxOn": {}, @@ -997,7 +1025,8 @@ "System.Composition.Hosting": { "StableVersions": [ "1.0.31", - "1.1.0" + "1.1.0", + "1.2.0" ], "BaselineVersion": "1.1.0", "InboxOn": {}, @@ -1010,7 +1039,8 @@ "System.Composition.Runtime": { "StableVersions": [ "1.0.31", - "1.1.0" + "1.1.0", + "1.2.0" ], "BaselineVersion": "1.1.0", "InboxOn": {}, @@ -1023,7 +1053,8 @@ "System.Composition.TypedParts": { "StableVersions": [ "1.0.31", - "1.1.0" + "1.1.0", + "1.2.0" ], "BaselineVersion": "1.1.0", "InboxOn": {}, @@ -1042,7 +1073,8 @@ }, "System.Configuration.ConfigurationManager": { "StableVersions": [ - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": {}, @@ -1149,6 +1181,9 @@ } }, "System.Data.DataSetExtensions": { + "StableVersions": [ + "4.5.0" + ], "InboxOn": { "net45": "4.0.0.0" }, @@ -1173,6 +1208,9 @@ } }, "System.Data.Odbc": { + "StableVersions": [ + "4.5.0" + ], "InboxOn": {}, "AssemblyVersionInPackageVersion": { "4.0.0.0": "4.5.0" @@ -1208,7 +1246,8 @@ "StableVersions": [ "4.1.0", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": { @@ -1337,7 +1376,8 @@ "StableVersions": [ "4.0.0", "4.3.0", - "4.4.1" + "4.4.1", + "4.5.0" ], "BaselineVersion": "4.4.1", "InboxOn": { @@ -1353,6 +1393,9 @@ } }, "System.Diagnostics.EventLog": { + "StableVersions": [ + "4.5.0" + ], "InboxOn": {}, "AssemblyVersionInPackageVersion": { "4.0.0.0": "4.5.0" @@ -1383,6 +1426,9 @@ } }, "System.Diagnostics.PerformanceCounter": { + "StableVersions": [ + "4.5.0" + ], "InboxOn": { "monoandroid10": "Any", "monotouch10": "Any", @@ -1588,6 +1634,9 @@ } }, "System.DirectoryServices": { + "StableVersions": [ + "4.5.0" + ], "InboxOn": { "net45": "4.0.0.0" }, @@ -1596,6 +1645,9 @@ } }, "System.DirectoryServices.AccountManagement": { + "StableVersions": [ + "4.5.0" + ], "InboxOn": { "net45": "4.0.0.0" }, @@ -1604,6 +1656,9 @@ } }, "System.DirectoryServices.Protocols": { + "StableVersions": [ + "4.5.0" + ], "InboxOn": { "net45": "4.0.0.0" }, @@ -1620,6 +1675,9 @@ } }, "System.Drawing.Common": { + "StableVersions": [ + "4.5.0" + ], "InboxOn": { "monoandroid10": "Any", "monotouch10": "Any", @@ -1985,7 +2043,8 @@ "StableVersions": [ "4.0.0", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": {}, @@ -2129,7 +2188,8 @@ "StableVersions": [ "4.0.0", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": {}, @@ -2141,6 +2201,9 @@ } }, "System.IO.Pipelines": { + "StableVersions": [ + "4.5.0" + ], "InboxOn": {}, "AssemblyVersionInPackageVersion": { "4.0.0.0": "4.5.0" @@ -2167,7 +2230,8 @@ "System.IO.Pipes.AccessControl": { "StableVersions": [ "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": { @@ -2181,7 +2245,8 @@ }, "System.IO.Ports": { "StableVersions": [ - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": { @@ -2220,7 +2285,8 @@ }, "System.Json": { "StableVersions": [ - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": { @@ -2393,6 +2459,9 @@ } }, "System.Management": { + "StableVersions": [ + "4.5.0" + ], "InboxOn": { "net45": "4.0.0.0" }, @@ -2406,6 +2475,9 @@ } }, "System.Memory": { + "StableVersions": [ + "4.5.0" + ], "InboxOn": { "netcoreapp2.1": "4.1.0.0", "monoandroid10": "Any", @@ -2548,7 +2620,8 @@ "4.0.0", "4.0.1", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": {}, @@ -2928,6 +3001,9 @@ } }, "System.Net.WebSockets.WebSocketProtocol": { + "StableVersions": [ + "4.5.0" + ], "InboxOn": {}, "AssemblyVersionInPackageVersion": { "4.0.0.0": "4.5.0" @@ -2960,7 +3036,8 @@ "4.1.0", "4.1.1", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": { @@ -3151,7 +3228,8 @@ "4.0.0", "4.0.1", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": { @@ -3291,7 +3369,8 @@ "1.1.0", "1.2.0", "1.4.1", - "1.5.0" + "1.5.0", + "1.6.0" ], "BaselineVersion": "1.5.0", "InboxOn": { @@ -3354,7 +3433,8 @@ "4.0.0", "4.1.0", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": { @@ -3511,6 +3591,9 @@ } }, "System.Runtime.Caching": { + "StableVersions": [ + "4.5.0" + ], "InboxOn": { "net45": "4.0.0.0", "monoandroid10": "Any", @@ -3525,6 +3608,9 @@ } }, "System.Runtime.CompilerServices.Unsafe": { + "StableVersions": [ + "4.5.0" + ], "InboxOn": { "uap10.0.16300": "4.0.4.0" }, @@ -4038,7 +4124,8 @@ "StableVersions": [ "4.0.0", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": { @@ -4107,7 +4194,8 @@ "StableVersions": [ "4.2.0", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": { @@ -4180,7 +4268,8 @@ "StableVersions": [ "4.0.0", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": {}, @@ -4195,7 +4284,8 @@ "StableVersions": [ "4.0.0", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": {}, @@ -4234,7 +4324,8 @@ "StableVersions": [ "4.0.0", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": { @@ -4279,7 +4370,8 @@ }, "System.Security.Cryptography.Xml": { "StableVersions": [ - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": {}, @@ -4290,7 +4382,8 @@ }, "System.Security.Permissions": { "StableVersions": [ - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": {}, @@ -4343,7 +4436,8 @@ "StableVersions": [ "4.0.0", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": { @@ -4517,6 +4611,9 @@ } }, "System.ServiceModel.Syndication": { + "StableVersions": [ + "4.5.0" + ], "InboxOn": {}, "AssemblyVersionInPackageVersion": { "4.0.0.0": "4.5.0" @@ -4560,7 +4657,8 @@ "StableVersions": [ "4.1.0", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": {}, @@ -4624,7 +4722,8 @@ "4.0.0", "4.0.1", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": { @@ -4691,7 +4790,8 @@ "StableVersions": [ "4.0.0", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": {}, @@ -4793,7 +4893,8 @@ "StableVersions": [ "4.0.0", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": {}, @@ -4805,6 +4906,9 @@ } }, "System.Threading.Channels": { + "StableVersions": [ + "4.5.0" + ], "InboxOn": {}, "AssemblyVersionInPackageVersion": { "4.0.0.0": "4.5.0" @@ -4878,7 +4982,8 @@ "4.6.0", "4.5.25", "4.7.0", - "4.8.0" + "4.8.0", + "4.9.0" ], "BaselineVersion": "4.8.0", "InboxOn": { @@ -4898,7 +5003,8 @@ "StableVersions": [ "4.0.0", "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": { @@ -5048,7 +5154,8 @@ "System.ValueTuple": { "StableVersions": [ "4.3.0", - "4.4.0" + "4.4.0", + "4.5.0" ], "BaselineVersion": "4.4.0", "InboxOn": { diff --git a/src/packages.builds b/src/packages.builds index 41d3c937b70e..aaf7881718f5 100644 --- a/src/packages.builds +++ b/src/packages.builds @@ -4,9 +4,10 @@ $(BinDir)pkg/reports/ + false - + $(AdditionalProperties) @@ -15,6 +16,16 @@ + + + $(AdditionalProperties) + + + $(AdditionalProperties) + + + + From 59b1d9a24cc0db66ccfac1363f2bdd264c1d5058 Mon Sep 17 00:00:00 2001 From: William Godbe Date: Tue, 3 Jul 2018 12:44:59 -0700 Subject: [PATCH 02/18] Port 'Delay console CPR timer once protocol has worked' to 2.1 (#30789) --- .../src/System/ConsolePal.Unix.cs | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/System.Console/src/System/ConsolePal.Unix.cs b/src/System.Console/src/System/ConsolePal.Unix.cs index 326a14e2b098..855aa55519ac 100644 --- a/src/System.Console/src/System/ConsolePal.Unix.cs +++ b/src/System.Console/src/System/ConsolePal.Unix.cs @@ -332,6 +332,13 @@ public static int CursorTop } } + /// + /// Tracks whether we've ever successfully received a response to a cursor position request (CPR). + /// If we have, then we can be more aggressive about expecting a response to subsequent requests, + /// e.g. using a longer timeout. + /// + private static bool s_everReceivedCursorPositionResponse; + /// Gets the current cursor position. This involves both writing to stdout and reading stdin. private static unsafe void GetCursorPosition(out int left, out int top) { @@ -358,7 +365,15 @@ private static unsafe void GetCursorPosition(out int left, out int top) // one thread's get_CursorLeft/Top from providing input to the other's Console.Read*. lock (StdInReader) { - Interop.Sys.InitializeConsoleBeforeRead(minChars: 0, decisecondsTimeout: 10); + // Because the CPR request/response protocol involves blocking until we get a certain + // response from the terminal, we want to avoid doing so if we don't know the terminal + // will definitely response. As such, we start with minChars == 0, which causes the + // terminal's read timer to start immediately. Once we've received a response for + // a request such that we know the terminal supports the protocol, we then specify + // minChars == 1. With that, the timer won't start until the first character is + // received. This makes the mechanism more reliable when there are high latencies + // involved in reading/writing, such as when accessing a remote system. + Interop.Sys.InitializeConsoleBeforeRead(minChars: (byte)(s_everReceivedCursorPositionResponse ? 1 : 0), decisecondsTimeout: 10); try { // Write out the cursor position report request. @@ -425,6 +440,9 @@ private static unsafe void GetCursorPosition(out int left, out int top) // else back into the StdInReader. ReadRowOrCol(bracketPos, semiPos, r, readBytes, ref top); ReadRowOrCol(semiPos, rPos, r, readBytes, ref left); + + // Mark that we've successfully received a CPR response at least once. + s_everReceivedCursorPositionResponse = true; } finally { From 28506b2f8ae790c4aa4b23bc45ece86a763d45dc Mon Sep 17 00:00:00 2001 From: Tomas Weinfurt Date: Tue, 3 Jul 2018 14:26:24 -0700 Subject: [PATCH 03/18] fix dereferencing uninitialized byte array in case recv() fails (#30312) (#30676) --- .../src/System/Net/Sockets/SocketPal.Unix.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/System.Net.Sockets/src/System/Net/Sockets/SocketPal.Unix.cs b/src/System.Net.Sockets/src/System/Net/Sockets/SocketPal.Unix.cs index da0bab6b1bc6..2f1f97c55be3 100644 --- a/src/System.Net.Sockets/src/System/Net/Sockets/SocketPal.Unix.cs +++ b/src/System.Net.Sockets/src/System/Net/Sockets/SocketPal.Unix.cs @@ -380,14 +380,15 @@ private static unsafe int ReceiveMessageFrom(SafeCloseSocket socket, SocketFlags receivedFlags = messageHeader.Flags; sockAddrLen = messageHeader.SocketAddressLen; - ipPacketInformation = GetIPPacketInformation(&messageHeader, isIPv4, isIPv6); } if (errno != Interop.Error.SUCCESS) { + ipPacketInformation = default(IPPacketInformation); return -1; } + ipPacketInformation = GetIPPacketInformation(&messageHeader, isIPv4, isIPv6); socketAddressLen = sockAddrLen; return checked((int)received); } @@ -442,15 +443,16 @@ private static unsafe int ReceiveMessageFrom( receivedFlags = messageHeader.Flags; int sockAddrLen = messageHeader.SocketAddressLen; - ipPacketInformation = GetIPPacketInformation(&messageHeader, isIPv4, isIPv6); if (errno == Interop.Error.SUCCESS) { + ipPacketInformation = GetIPPacketInformation(&messageHeader, isIPv4, isIPv6); socketAddressLen = sockAddrLen; return checked((int)received); } else { + ipPacketInformation = default(IPPacketInformation); return -1; } } From abb7213665a40d264e4b5b2201f3d1c54079ac61 Mon Sep 17 00:00:00 2001 From: Wes Haggard Date: Tue, 3 Jul 2018 16:39:31 -0700 Subject: [PATCH 04/18] Allows include build number for packages blocked from going stable For any packages that we doin't want to ship and marked as BlockStable we should always force include the label and build number in the package name. --- dir.targets | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dir.targets b/dir.targets index 2bf1e4371d35..5bc52a85093a 100644 --- a/dir.targets +++ b/dir.targets @@ -93,6 +93,15 @@ + + + true + true + + From 8f71a67bff5520b6fff90703d2a3e73641bc0406 Mon Sep 17 00:00:00 2001 From: Gleb Balykov Date: Thu, 5 Jul 2018 21:46:35 +0300 Subject: [PATCH 05/18] Add missing RIDs for Tizen (#30640) --- .../runtime.compatibility.json | 162 ++++++++++++++++++ pkg/Microsoft.NETCore.Platforms/runtime.json | 66 +++++++ .../runtimeGroups.props | 4 +- 3 files changed, 230 insertions(+), 2 deletions(-) diff --git a/pkg/Microsoft.NETCore.Platforms/runtime.compatibility.json b/pkg/Microsoft.NETCore.Platforms/runtime.compatibility.json index b65524646390..1c537ddfe56a 100644 --- a/pkg/Microsoft.NETCore.Platforms/runtime.compatibility.json +++ b/pkg/Microsoft.NETCore.Platforms/runtime.compatibility.json @@ -4641,6 +4641,33 @@ "any", "base" ], + "tizen-x86": [ + "tizen-x86", + "tizen", + "linux-x86", + "linux", + "unix-x86", + "unix", + "any", + "base" + ], + "tizen-x86-corert": [ + "tizen-x86-corert", + "tizen-corert", + "tizen-x86", + "linux-x86-corert", + "tizen", + "linux-corert", + "linux-x86", + "unix-x86-corert", + "linux", + "unix-corert", + "unix-x86", + "unix", + "corert", + "any", + "base" + ], "tizen.4.0.0": [ "tizen.4.0.0", "tizen", @@ -4695,6 +4722,141 @@ "any", "base" ], + "tizen.4.0.0-x86": [ + "tizen.4.0.0-x86", + "tizen.4.0.0", + "tizen-x86", + "tizen", + "linux-x86", + "linux", + "unix-x86", + "unix", + "any", + "base" + ], + "tizen.4.0.0-x86-corert": [ + "tizen.4.0.0-x86-corert", + "tizen.4.0.0-corert", + "tizen.4.0.0-x86", + "tizen.4.0.0", + "tizen-x86-corert", + "tizen-corert", + "tizen-x86", + "tizen", + "linux-x86-corert", + "linux-corert", + "linux-x86", + "linux", + "unix-x86-corert", + "unix-corert", + "unix-x86", + "unix", + "corert", + "any", + "base" + ], + "tizen.5.0.0": [ + "tizen.5.0.0", + "tizen.4.0.0", + "tizen", + "linux", + "unix", + "any", + "base" + ], + "tizen.5.0.0-armel": [ + "tizen.5.0.0-armel", + "tizen.5.0.0", + "tizen.4.0.0-armel", + "tizen.4.0.0", + "tizen-armel", + "tizen", + "linux-armel", + "linux", + "unix-armel", + "unix", + "any", + "base" + ], + "tizen.5.0.0-armel-corert": [ + "tizen.5.0.0-armel-corert", + "tizen.5.0.0-corert", + "tizen.5.0.0-armel", + "tizen.5.0.0", + "tizen.4.0.0-armel-corert", + "tizen.4.0.0-corert", + "tizen.4.0.0-armel", + "tizen.4.0.0", + "tizen-armel-corert", + "tizen-corert", + "tizen-armel", + "tizen", + "linux-armel-corert", + "linux-corert", + "linux-armel", + "linux", + "unix-armel-corert", + "unix-corert", + "unix-armel", + "unix", + "corert", + "any", + "base" + ], + "tizen.5.0.0-corert": [ + "tizen.5.0.0-corert", + "tizen.5.0.0", + "tizen.4.0.0-corert", + "tizen.4.0.0", + "tizen-corert", + "tizen", + "linux-corert", + "linux", + "unix-corert", + "unix", + "corert", + "any", + "base" + ], + "tizen.5.0.0-x86": [ + "tizen.5.0.0-x86", + "tizen.5.0.0", + "tizen.4.0.0-x86", + "tizen.4.0.0", + "tizen-x86", + "tizen", + "linux-x86", + "linux", + "unix-x86", + "unix", + "any", + "base" + ], + "tizen.5.0.0-x86-corert": [ + "tizen.5.0.0-x86-corert", + "tizen.5.0.0-corert", + "tizen.5.0.0-x86", + "tizen.5.0.0", + "tizen.4.0.0-x86-corert", + "tizen.4.0.0-corert", + "tizen.4.0.0-x86", + "tizen.4.0.0", + "tizen-x86-corert", + "tizen-corert", + "tizen-x86", + "tizen", + "linux-x86-corert", + "linux-corert", + "linux-x86", + "linux", + "unix-x86-corert", + "unix-corert", + "unix-x86", + "unix", + "corert", + "any", + "base" + ], "ubuntu": [ "ubuntu", "debian", diff --git a/pkg/Microsoft.NETCore.Platforms/runtime.json b/pkg/Microsoft.NETCore.Platforms/runtime.json index 30f99f8e58e1..81cf1c7d0e0c 100644 --- a/pkg/Microsoft.NETCore.Platforms/runtime.json +++ b/pkg/Microsoft.NETCore.Platforms/runtime.json @@ -1817,6 +1817,19 @@ "linux-corert" ] }, + "tizen-x86": { + "#import": [ + "tizen", + "linux-x86" + ] + }, + "tizen-x86-corert": { + "#import": [ + "tizen-corert", + "tizen-x86", + "linux-x86-corert" + ] + }, "tizen.4.0.0": { "#import": [ "tizen" @@ -1842,6 +1855,59 @@ "tizen-corert" ] }, + "tizen.4.0.0-x86": { + "#import": [ + "tizen.4.0.0", + "tizen-x86" + ] + }, + "tizen.4.0.0-x86-corert": { + "#import": [ + "tizen.4.0.0-corert", + "tizen.4.0.0-x86", + "tizen.4.0.0", + "tizen-x86-corert" + ] + }, + "tizen.5.0.0": { + "#import": [ + "tizen.4.0.0" + ] + }, + "tizen.5.0.0-armel": { + "#import": [ + "tizen.5.0.0", + "tizen.4.0.0-armel" + ] + }, + "tizen.5.0.0-armel-corert": { + "#import": [ + "tizen.5.0.0-corert", + "tizen.5.0.0-armel", + "tizen.5.0.0", + "tizen.4.0.0-armel-corert" + ] + }, + "tizen.5.0.0-corert": { + "#import": [ + "tizen.5.0.0", + "tizen.4.0.0-corert" + ] + }, + "tizen.5.0.0-x86": { + "#import": [ + "tizen.5.0.0", + "tizen.4.0.0-x86" + ] + }, + "tizen.5.0.0-x86-corert": { + "#import": [ + "tizen.5.0.0-corert", + "tizen.5.0.0-x86", + "tizen.5.0.0", + "tizen.4.0.0-x86-corert" + ] + }, "ubuntu": { "#import": [ "debian" diff --git a/pkg/Microsoft.NETCore.Platforms/runtimeGroups.props b/pkg/Microsoft.NETCore.Platforms/runtimeGroups.props index c8898ed0bb49..6cee9f7aa10b 100644 --- a/pkg/Microsoft.NETCore.Platforms/runtimeGroups.props +++ b/pkg/Microsoft.NETCore.Platforms/runtimeGroups.props @@ -111,8 +111,8 @@ linux - armel - 4.0.0 + x86;armel + 4.0.0;5.0.0 From 87b79bde5fb6f6a7382104d9d4d954a259181114 Mon Sep 17 00:00:00 2001 From: Maryam Ariyan Date: Thu, 5 Jul 2018 15:00:37 -0400 Subject: [PATCH 06/18] Fix compilation for deprecated API on macOS Mojave preview (#30716) (#30744) * Fix compilation for deprecated API on macOS Mojave preview Fixes: #30599 --- .../System.Security.Cryptography.Native.Apple/CMakeLists.txt | 3 +++ .../pal_x509chain.cpp | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Native/Unix/System.Security.Cryptography.Native.Apple/CMakeLists.txt b/src/Native/Unix/System.Security.Cryptography.Native.Apple/CMakeLists.txt index 0ed02c6e9dfe..5049a5f4b7dc 100644 --- a/src/Native/Unix/System.Security.Cryptography.Native.Apple/CMakeLists.txt +++ b/src/Native/Unix/System.Security.Cryptography.Native.Apple/CMakeLists.txt @@ -26,6 +26,9 @@ set(NATIVECRYPTO_SOURCES pal_x509chain.cpp ) +# Temporary workaround for dotnet/corefx issue #30599 +add_compile_options(-Wno-deprecated-declarations) + add_library(System.Security.Cryptography.Native.Apple SHARED ${NATIVECRYPTO_SOURCES} diff --git a/src/Native/Unix/System.Security.Cryptography.Native.Apple/pal_x509chain.cpp b/src/Native/Unix/System.Security.Cryptography.Native.Apple/pal_x509chain.cpp index b84540339368..2d8cae14cc4a 100644 --- a/src/Native/Unix/System.Security.Cryptography.Native.Apple/pal_x509chain.cpp +++ b/src/Native/Unix/System.Security.Cryptography.Native.Apple/pal_x509chain.cpp @@ -186,7 +186,7 @@ static void MergeStatusCodes(CFTypeRef key, CFTypeRef value, void* context) // It doesn't represent a new error code, and we're still getting the old ones, so // just ignore it for now. } - else if (CFEqual(keyString, CFSTR("NonEmptySubject"))) + else if (CFEqual(keyString, CFSTR("NonEmptySubject")) || CFEqual(keyString, CFSTR("GrayListedKey"))) { // Not a "problem" that we report. } From 2185b5512232ace2e60b44b6789b5e8a8320fe98 Mon Sep 17 00:00:00 2001 From: William Godbe Date: Thu, 5 Jul 2018 16:04:50 -0700 Subject: [PATCH 07/18] Bump Platforms package version & build it (#30854) --- .../Microsoft.NETCore.Platforms.pkgproj | 2 +- src/packages.builds | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/Microsoft.NETCore.Platforms/Microsoft.NETCore.Platforms.pkgproj b/pkg/Microsoft.NETCore.Platforms/Microsoft.NETCore.Platforms.pkgproj index 22db2c447bbf..623230959ff4 100644 --- a/pkg/Microsoft.NETCore.Platforms/Microsoft.NETCore.Platforms.pkgproj +++ b/pkg/Microsoft.NETCore.Platforms/Microsoft.NETCore.Platforms.pkgproj @@ -3,7 +3,7 @@ - 2.1.0 + 2.1.1 true false diff --git a/src/packages.builds b/src/packages.builds index 1a6d648d1831..3cd400a6f86c 100644 --- a/src/packages.builds +++ b/src/packages.builds @@ -33,6 +33,9 @@ $(AdditionalProperties) + + $(AdditionalProperties) + From 67ee641dc3681681832320d90ead675b038fadf2 Mon Sep 17 00:00:00 2001 From: dotnet-maestro-bot Date: Sun, 8 Jul 2018 06:09:47 -0700 Subject: [PATCH 08/18] Update BuildTools, CoreFx to rc1-03006-01, servicing-26703-08, respectively (#30884) --- BuildToolsVersion.txt | 2 +- dependencies.props | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/BuildToolsVersion.txt b/BuildToolsVersion.txt index eb6df085b9a3..6b4d62514170 100644 --- a/BuildToolsVersion.txt +++ b/BuildToolsVersion.txt @@ -1 +1 @@ -2.1.0-rc1-02913-01 +2.1.0-rc1-03006-01 diff --git a/dependencies.props b/dependencies.props index 43877884e1fa..491aaf5d90f2 100644 --- a/dependencies.props +++ b/dependencies.props @@ -9,7 +9,7 @@ These ref versions are pulled from https://github.com/dotnet/versions. --> - 11f431e6bd908d5fdd94c187d2741d9c40a4287d + 8fc22d261883eb7a1f8765bfe716d806303d0318 11f431e6bd908d5fdd94c187d2741d9c40a4287d 11f431e6bd908d5fdd94c187d2741d9c40a4287d 96dc7805f5df4a70a55783964ce69dcd91bfca80 @@ -17,7 +17,7 @@ ccd922b62227c43ed2dac6bcb737321dd2b07be0 8bd1ec5fac9f0eec34ff6b34b1d878b4359e02dd c520a2569b40fc53cf51e4f6970c3e7411adc173 - d1e2bedf58a4a66aadda4f5751ecdaf3ad4f62fe + 8fc22d261883eb7a1f8765bfe716d806303d0318 @@ -31,7 +31,7 @@ - servicing + servicing-26703-08 2.1.0 2.1.1-servicing-26606-02 beta-26413-00 @@ -47,7 +47,7 @@ 1.0.3-prerelease-00921-01 1.0.0-beta-build0019 2.0.5 - 2.1.0-rc1-02913-01 + 2.1.0-rc1-03006-01 2.0.0-rc-61101-17 @@ -56,7 +56,7 @@ Microsoft.DotNet.Build.Tasks.Feed - 2.1.0-rc1-02913-01 + 2.1.0-rc1-03006-01 From ffa1879e0df79299979696421f31a8b18b606881 Mon Sep 17 00:00:00 2001 From: Gleb Balykov Date: Tue, 10 Jul 2018 20:52:18 +0300 Subject: [PATCH 09/18] Add Tizen 5.0 RID for NETCoreApp (#30888) --- pkg/Microsoft.Private.CoreFx.NETCoreApp/netcoreapp.rids.props | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/Microsoft.Private.CoreFx.NETCoreApp/netcoreapp.rids.props b/pkg/Microsoft.Private.CoreFx.NETCoreApp/netcoreapp.rids.props index 06bdbc769641..520c72a119ef 100644 --- a/pkg/Microsoft.Private.CoreFx.NETCoreApp/netcoreapp.rids.props +++ b/pkg/Microsoft.Private.CoreFx.NETCoreApp/netcoreapp.rids.props @@ -28,5 +28,8 @@ armel + + armel + From 2f50f84779a40c34d32ef3b57bbe46ee767fffc6 Mon Sep 17 00:00:00 2001 From: dotnet-maestro-bot Date: Tue, 10 Jul 2018 12:31:55 -0700 Subject: [PATCH 10/18] Update CoreClr, CoreFx, CoreSetup to servicing-26708-02, servicing-26708-02, servicing-26708-02, respectively --- dependencies.props | 18 +++++++++--------- tools-local/ILAsmVersion.txt | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dependencies.props b/dependencies.props index 491aaf5d90f2..18e141d2783b 100644 --- a/dependencies.props +++ b/dependencies.props @@ -9,9 +9,9 @@ These ref versions are pulled from https://github.com/dotnet/versions. --> - 8fc22d261883eb7a1f8765bfe716d806303d0318 - 11f431e6bd908d5fdd94c187d2741d9c40a4287d - 11f431e6bd908d5fdd94c187d2741d9c40a4287d + b933a1ea9efa451dc6e06da2920f1c93253160b2 + b933a1ea9efa451dc6e06da2920f1c93253160b2 + b933a1ea9efa451dc6e06da2920f1c93253160b2 96dc7805f5df4a70a55783964ce69dcd91bfca80 ccd922b62227c43ed2dac6bcb737321dd2b07be0 ccd922b62227c43ed2dac6bcb737321dd2b07be0 @@ -31,15 +31,15 @@ - servicing-26703-08 - 2.1.0 - 2.1.1-servicing-26606-02 + servicing-26708-02 + 2.1.1-servicing-26708-02 + 2.1.3-servicing-26708-02 beta-26413-00 beta-26413-00 1.0.0-beta-26413-00 - 2.1.1 - 2.1.1 - 2.1.1 + 2.1.3-servicing-26708-02 + 2.1.3-servicing-26708-02 + 2.1.3-servicing-26708-02 4.4.0 diff --git a/tools-local/ILAsmVersion.txt b/tools-local/ILAsmVersion.txt index ec43494dd155..09640f58db66 100644 --- a/tools-local/ILAsmVersion.txt +++ b/tools-local/ILAsmVersion.txt @@ -1 +1 @@ -2.1.1-servicing-26606-02 \ No newline at end of file +2.1.3-servicing-26708-02 \ No newline at end of file From 35a009a17866ce41a95f25780503b82183b5ec9e Mon Sep 17 00:00:00 2001 From: Max Kerr Date: Tue, 10 Jul 2018 16:48:05 -0700 Subject: [PATCH 11/18] Add support and tests for HTTP 308 Permanent Redirect (#30398) (#30954) This change adds support for 308 redirects to SocketsHttpHandler, and enables 308 redirects in our tests. Fixes: #30389 --- .../HttpHandlerDiagnosticListener.cs | 11 +++--- .../SocketsHttpHandler/RedirectHandler.cs | 1 + .../src/uap/System/Net/HttpHandlerToFilter.cs | 8 ++-- .../FunctionalTests/HttpClientHandlerTest.cs | 37 ++++++++++++++++++- 4 files changed, 48 insertions(+), 9 deletions(-) diff --git a/src/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/HttpHandlerDiagnosticListener.cs b/src/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/HttpHandlerDiagnosticListener.cs index ec8feba0bcb5..bbfb87c05521 100644 --- a/src/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/HttpHandlerDiagnosticListener.cs +++ b/src/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/HttpHandlerDiagnosticListener.cs @@ -653,11 +653,12 @@ private bool IsLastResponse(HttpWebRequest request, HttpStatusCode statusCode) { if (request.AllowAutoRedirect) { - if (statusCode == HttpStatusCode.Ambiguous || // 300 - statusCode == HttpStatusCode.Moved || // 301 - statusCode == HttpStatusCode.Redirect || // 302 - statusCode == HttpStatusCode.RedirectMethod || // 303 - statusCode == HttpStatusCode.RedirectKeepVerb) // 307 + if (statusCode == HttpStatusCode.Ambiguous || // 300 + statusCode == HttpStatusCode.Moved || // 301 + statusCode == HttpStatusCode.Redirect || // 302 + statusCode == HttpStatusCode.RedirectMethod || // 303 + statusCode == HttpStatusCode.RedirectKeepVerb || // 307 + (int)statusCode == 308) // 308 Permanent Redirect is not in netfx yet, and so has to be specified this way. { return s_autoRedirectsAccessor(request) >= request.MaximumAutomaticRedirections; } diff --git a/src/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/RedirectHandler.cs b/src/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/RedirectHandler.cs index ede57cf9ca56..138bac21db78 100644 --- a/src/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/RedirectHandler.cs +++ b/src/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/RedirectHandler.cs @@ -81,6 +81,7 @@ private Uri GetUriForRedirect(Uri requestUri, HttpResponseMessage response) case HttpStatusCode.SeeOther: case HttpStatusCode.TemporaryRedirect: case HttpStatusCode.MultipleChoices: + case HttpStatusCode.PermanentRedirect: break; default: diff --git a/src/System.Net.Http/src/uap/System/Net/HttpHandlerToFilter.cs b/src/System.Net.Http/src/uap/System/Net/HttpHandlerToFilter.cs index 53e7b9b2bf20..693955680df3 100644 --- a/src/System.Net.Http/src/uap/System/Net/HttpHandlerToFilter.cs +++ b/src/System.Net.Http/src/uap/System/Net/HttpHandlerToFilter.cs @@ -116,7 +116,8 @@ protected internal override async Task SendAsync(HttpReques response.StatusCode != HttpStatusCode.MovedPermanently && response.StatusCode != HttpStatusCode.Redirect && response.StatusCode != HttpStatusCode.RedirectMethod && - response.StatusCode != HttpStatusCode.RedirectKeepVerb) + response.StatusCode != HttpStatusCode.RedirectKeepVerb && + response.StatusCode != HttpStatusCode.PermanentRedirect) { break; } @@ -151,10 +152,11 @@ protected internal override async Task SendAsync(HttpReques } // Follow HTTP RFC 7231 rules. In general, 3xx responses - // except for 307 will keep verb except POST becomes GET. - // 307 responses have all verbs stay the same. + // except for 307 and 308 will keep verb except POST becomes GET. + // 307 and 308 responses have all verbs stay the same. // https://tools.ietf.org/html/rfc7231#section-6.4 if (response.StatusCode != HttpStatusCode.RedirectKeepVerb && + response.StatusCode != HttpStatusCode.PermanentRedirect && requestHttpMethod == HttpMethod.Post) { requestHttpMethod = HttpMethod.Get; diff --git a/src/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.cs b/src/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.cs index 2d7a9c55b721..14960a11ea29 100644 --- a/src/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.cs +++ b/src/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.cs @@ -62,7 +62,8 @@ public abstract class HttpClientHandlerTest : HttpClientTestBase new object[] { 301 }, new object[] { 302 }, new object[] { 303 }, - new object[] { 307 } + new object[] { 307 }, + new object[] { 308 } }; public static readonly object[][] RedirectStatusCodesOldMethodsNewMethods = { @@ -85,6 +86,10 @@ public abstract class HttpClientHandlerTest : HttpClientTestBase new object[] { 307, "GET", "GET" }, new object[] { 307, "POST", "POST" }, new object[] { 307, "HEAD", "HEAD" }, + + new object[] { 308, "GET", "GET" }, + new object[] { 308, "POST", "POST" }, + new object[] { 308, "HEAD", "HEAD" }, }; // Standard HTTP methods defined in RFC7231: http://tools.ietf.org/html/rfc7231#section-4.3 @@ -245,6 +250,12 @@ public async Task UseDefaultCredentials_SetToFalseAndServerNeedsAuth_StatusCodeU [Theory, MemberData(nameof(RedirectStatusCodes))] public async Task DefaultHeaders_SetCredentials_ClearedOnRedirect(int statusCode) { + if (statusCode == 308 && (PlatformDetection.IsFullFramework || IsWinHttpHandler && PlatformDetection.WindowsVersion < 10)) + { + // 308 redirects are not supported on old versions of WinHttp, or on .NET Framework. + return; + } + HttpClientHandler handler = CreateHttpClientHandler(); using (var client = new HttpClient(handler)) { @@ -721,6 +732,12 @@ await LoopbackServer.CreateServerAsync(async (server, url) => [Theory, MemberData(nameof(RedirectStatusCodes))] public async Task GetAsync_AllowAutoRedirectFalse_RedirectFromHttpToHttp_StatusCodeRedirect(int statusCode) { + if (statusCode == 308 && (PlatformDetection.IsFullFramework || IsWinHttpHandler && PlatformDetection.WindowsVersion < 10)) + { + // 308 redirects are not supported on old versions of WinHttp, or on .NET Framework. + return; + } + HttpClientHandler handler = CreateHttpClientHandler(); handler.AllowAutoRedirect = false; using (var client = new HttpClient(handler)) @@ -750,6 +767,12 @@ public async Task AllowAutoRedirect_True_ValidateNewMethodUsedOnRedirection( newMethod = "POST"; } + if (statusCode == 308 && (PlatformDetection.IsFullFramework || IsWinHttpHandler && PlatformDetection.WindowsVersion < 10)) + { + // 308 redirects are not supported on old versions of WinHttp, or on .NET Framework. + return; + } + HttpClientHandler handler = CreateHttpClientHandler(); using (var client = new HttpClient(handler)) { @@ -868,6 +891,12 @@ await LoopbackServer.CreateServerAsync(async (redirServer, redirUrl) => [Theory, MemberData(nameof(RedirectStatusCodes))] public async Task GetAsync_AllowAutoRedirectTrue_RedirectFromHttpToHttp_StatusCodeOK(int statusCode) { + if (statusCode == 308 && (PlatformDetection.IsFullFramework || IsWinHttpHandler && PlatformDetection.WindowsVersion < 10)) + { + // 308 redirects are not supported on old versions of WinHttp, or on .NET Framework. + return; + } + HttpClientHandler handler = CreateHttpClientHandler(); handler.AllowAutoRedirect = true; using (var client = new HttpClient(handler)) @@ -1225,6 +1254,12 @@ public async Task HttpClientHandler_CredentialIsNotCredentialCacheAfterRedirect_ [Theory, MemberData(nameof(RedirectStatusCodes))] public async Task GetAsync_CredentialIsCredentialCacheUriRedirect_StatusCodeOK(int statusCode) { + if (statusCode == 308 && (PlatformDetection.IsFullFramework || IsWinHttpHandler && PlatformDetection.WindowsVersion < 10)) + { + // 308 redirects are not supported on old versions of WinHttp, or on .NET Framework. + return; + } + Uri uri = Configuration.Http.BasicAuthUriForCreds(secure: false, userName: Username, password: Password); Uri redirectUri = Configuration.Http.RedirectUriForCreds( secure: false, From 9cec26d045f09bde63c902199547046d0cfe0170 Mon Sep 17 00:00:00 2001 From: Jeremy Kuhne Date: Tue, 10 Jul 2018 17:16:13 -0700 Subject: [PATCH 12/18] Port Tests for #30263 fix (#30374) (#30557) * Tests for #30263 fix Goes with https://github.com/dotnet/coreclr/pull/18460 * Add case for `C:\` to `D:\` --- .../tests/System/IO/Path.GetRelativePath.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/System.Runtime.Extensions/tests/System/IO/Path.GetRelativePath.cs b/src/System.Runtime.Extensions/tests/System/IO/Path.GetRelativePath.cs index 36a03d6bfc9f..3f48bfd9ccb5 100644 --- a/src/System.Runtime.Extensions/tests/System/IO/Path.GetRelativePath.cs +++ b/src/System.Runtime.Extensions/tests/System/IO/Path.GetRelativePath.cs @@ -17,12 +17,26 @@ public static class GetRelativePathTests [InlineData(@"C:\", @"C:\b", @"b")] [InlineData(@"C:\a", @"C:\b", @"..\b")] [InlineData(@"C:\a", @"C:\b\", @"..\b\")] + [InlineData(@"C:\a\b", @"C:\a", @"..")] + [InlineData(@"C:\a\b", @"C:\a\", @"..")] + [InlineData(@"C:\a\b\", @"C:\a", @"..")] + [InlineData(@"C:\a\b\", @"C:\a\", @"..")] + [InlineData(@"C:\a\b\c", @"C:\a\b", @"..")] + [InlineData(@"C:\a\b\c", @"C:\a\b\", @"..")] + [InlineData(@"C:\a\b\c", @"C:\a", @"..\..")] + [InlineData(@"C:\a\b\c", @"C:\a\", @"..\..")] + [InlineData(@"C:\a\b\c\", @"C:\a\b", @"..")] + [InlineData(@"C:\a\b\c\", @"C:\a\b\", @"..")] + [InlineData(@"C:\a\b\c\", @"C:\a", @"..\..")] + [InlineData(@"C:\a\b\c\", @"C:\a\", @"..\..")] [InlineData(@"C:\a\", @"C:\b", @"..\b")] [InlineData(@"C:\a", @"C:\a\b", @"b")] [InlineData(@"C:\a", @"C:\A\b", @"b")] [InlineData(@"C:\a", @"C:\b\c", @"..\b\c")] [InlineData(@"C:\a\", @"C:\a\b", @"b")] + [InlineData(@"C:\", @"D:\", @"D:\")] [InlineData(@"C:\", @"D:\b", @"D:\b")] + [InlineData(@"C:\", @"D:\b\", @"D:\b\")] [InlineData(@"C:\a", @"D:\b", @"D:\b")] [InlineData(@"C:\a\", @"D:\b", @"D:\b")] [InlineData(@"C:\ab", @"C:\a", @"..\a")] From 8f34b492591366361a7a7d2c03824b200bbe161a Mon Sep 17 00:00:00 2001 From: Koundinya Veluri Date: Tue, 10 Jul 2018 19:27:43 -0700 Subject: [PATCH 13/18] Fix handle double-free in recently added WindowsIdentity test (#30731) (#30977) --- .../tests/WindowsIdentityTests.cs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/System.Security.Principal.Windows/tests/WindowsIdentityTests.cs b/src/System.Security.Principal.Windows/tests/WindowsIdentityTests.cs index f107533511c8..4d696137eb7f 100644 --- a/src/System.Security.Principal.Windows/tests/WindowsIdentityTests.cs +++ b/src/System.Security.Principal.Windows/tests/WindowsIdentityTests.cs @@ -129,12 +129,16 @@ public static void RunImpersonatedTest_InvalidHandle() { using (var mutex = new Mutex()) { - Assert.Throws(() => + SafeAccessTokenHandle handle = null; + try { - WindowsIdentity.RunImpersonated( - new SafeAccessTokenHandle(mutex.SafeWaitHandle.DangerousGetHandle()), - () => { }); - }); + handle = new SafeAccessTokenHandle(mutex.SafeWaitHandle.DangerousGetHandle()); + Assert.Throws(() => WindowsIdentity.RunImpersonated(handle, () => { })); + } + finally + { + handle?.SetHandleAsInvalid(); + } } } From 0652864034665f23e607b9c966155515ac301f97 Mon Sep 17 00:00:00 2001 From: Ricardo Arenas Date: Wed, 11 Jul 2018 11:52:31 -0700 Subject: [PATCH 14/18] Always run end.ps1/sh scripts --- buildpipeline/DotNet-CoreFx-Trusted-Linux-Crossbuild.json | 1 + buildpipeline/DotNet-CoreFx-Trusted-Linux.json | 1 + buildpipeline/DotNet-CoreFx-Trusted-OSX.json | 1 + buildpipeline/DotNet-CoreFx-Trusted-Windows-NoTest.json | 1 + buildpipeline/DotNet-CoreFx-Trusted-Windows.json | 1 + buildpipeline/DotNet-Trusted-Publish-Symbols.json | 1 + buildpipeline/DotNet-Trusted-Publish.json | 1 + buildpipeline/DotNet-Trusted-Tests-Publish.json | 1 + 8 files changed, 8 insertions(+) diff --git a/buildpipeline/DotNet-CoreFx-Trusted-Linux-Crossbuild.json b/buildpipeline/DotNet-CoreFx-Trusted-Linux-Crossbuild.json index 032c3fd655e1..639b6a6807c0 100644 --- a/buildpipeline/DotNet-CoreFx-Trusted-Linux-Crossbuild.json +++ b/buildpipeline/DotNet-CoreFx-Trusted-Linux-Crossbuild.json @@ -359,6 +359,7 @@ "continueOnError": true, "displayName": "run end.sh", "timeoutInMinutes": 0, + "condition": "always()", "alwaysRun": true, "task": { "id": "10f1f9a1-74b0-47ab-87bf-e3c9c68e8b0d", diff --git a/buildpipeline/DotNet-CoreFx-Trusted-Linux.json b/buildpipeline/DotNet-CoreFx-Trusted-Linux.json index f3ae7e29f031..95fc9ec4f72c 100644 --- a/buildpipeline/DotNet-CoreFx-Trusted-Linux.json +++ b/buildpipeline/DotNet-CoreFx-Trusted-Linux.json @@ -359,6 +359,7 @@ "continueOnError": true, "displayName": "run end.sh", "timeoutInMinutes": 0, + "condition": "always()", "alwaysRun": true, "task": { "id": "10f1f9a1-74b0-47ab-87bf-e3c9c68e8b0d", diff --git a/buildpipeline/DotNet-CoreFx-Trusted-OSX.json b/buildpipeline/DotNet-CoreFx-Trusted-OSX.json index 2c221dc98d31..bb8150136e08 100644 --- a/buildpipeline/DotNet-CoreFx-Trusted-OSX.json +++ b/buildpipeline/DotNet-CoreFx-Trusted-OSX.json @@ -275,6 +275,7 @@ "continueOnError": true, "displayName": "run end.sh", "timeoutInMinutes": 0, + "condition": "always()", "alwaysRun": true, "task": { "id": "10f1f9a1-74b0-47ab-87bf-e3c9c68e8b0d", diff --git a/buildpipeline/DotNet-CoreFx-Trusted-Windows-NoTest.json b/buildpipeline/DotNet-CoreFx-Trusted-Windows-NoTest.json index 3e740ace1014..a5fd16f19201 100644 --- a/buildpipeline/DotNet-CoreFx-Trusted-Windows-NoTest.json +++ b/buildpipeline/DotNet-CoreFx-Trusted-Windows-NoTest.json @@ -299,6 +299,7 @@ "alwaysRun": true, "displayName": "run end.ps1", "timeoutInMinutes": 0, + "condition": "always()", "task": { "id": "e213ff0f-5d5c-4791-802d-52ea3e7be1f1", "versionSpec": "2.*", diff --git a/buildpipeline/DotNet-CoreFx-Trusted-Windows.json b/buildpipeline/DotNet-CoreFx-Trusted-Windows.json index 93f4d0e57437..679aa965178b 100644 --- a/buildpipeline/DotNet-CoreFx-Trusted-Windows.json +++ b/buildpipeline/DotNet-CoreFx-Trusted-Windows.json @@ -350,6 +350,7 @@ "alwaysRun": true, "displayName": "run end.ps1", "timeoutInMinutes": 0, + "condition": "always()", "task": { "id": "e213ff0f-5d5c-4791-802d-52ea3e7be1f1", "versionSpec": "2.*", diff --git a/buildpipeline/DotNet-Trusted-Publish-Symbols.json b/buildpipeline/DotNet-Trusted-Publish-Symbols.json index 06dbfcfbfa95..22ee998f659d 100644 --- a/buildpipeline/DotNet-Trusted-Publish-Symbols.json +++ b/buildpipeline/DotNet-Trusted-Publish-Symbols.json @@ -149,6 +149,7 @@ "alwaysRun": true, "displayName": "run end.ps1", "timeoutInMinutes": 0, + "condition": "always()", "task": { "id": "e213ff0f-5d5c-4791-802d-52ea3e7be1f1", "versionSpec": "2.*", diff --git a/buildpipeline/DotNet-Trusted-Publish.json b/buildpipeline/DotNet-Trusted-Publish.json index 8facc2013abd..b771ca61de8c 100644 --- a/buildpipeline/DotNet-Trusted-Publish.json +++ b/buildpipeline/DotNet-Trusted-Publish.json @@ -298,6 +298,7 @@ "alwaysRun": true, "displayName": "run end.ps1", "timeoutInMinutes": 0, + "condition": "always()", "task": { "id": "e213ff0f-5d5c-4791-802d-52ea3e7be1f1", "versionSpec": "2.*", diff --git a/buildpipeline/DotNet-Trusted-Tests-Publish.json b/buildpipeline/DotNet-Trusted-Tests-Publish.json index 7b6d50b46262..a23e340afe21 100644 --- a/buildpipeline/DotNet-Trusted-Tests-Publish.json +++ b/buildpipeline/DotNet-Trusted-Tests-Publish.json @@ -144,6 +144,7 @@ "alwaysRun": true, "displayName": "run end.ps1", "timeoutInMinutes": 0, + "condition": "always()", "task": { "id": "e213ff0f-5d5c-4791-802d-52ea3e7be1f1", "versionSpec": "2.*", From 682b8d94795235ff2ae5d7454541666c53fa3539 Mon Sep 17 00:00:00 2001 From: Wes Haggard Date: Mon, 16 Jul 2018 09:18:45 -0700 Subject: [PATCH 15/18] Ensure we generate version prop before sync parallel build We need to generate the buildversion props file before we start a parallel build otherwise we race condition that can break the build. --- build.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.proj b/build.proj index b1a793952c03..3b71a9577403 100644 --- a/build.proj +++ b/build.proj @@ -42,7 +42,7 @@ - + From b6be5d7cf0e05f103614338d9aca77639f0644dd Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Tue, 17 Jul 2018 11:26:40 -0700 Subject: [PATCH 16/18] Ensure ConcurrentBag's TryTake is linearizable (#30947) (#31009) For .NET Core 2.0, I ported the ThreadPool's work-stealing implementation to ConcurrentBag, leading to significant performance throughput and allocation improvements. However, there's a subtle difference in the concurrency guarantees the ThreadPool's implementation provided from what ConcurrentBag needs, which ends up breaking certain usage patterns on top of ConcurrentBag. Specifically, ThreadPool's "steal" implementation need not be fully linearizable. It's possible for a thread to see the bag's count as 1, and then while the thread is doing a take/steal for its count to never drop below 1, but for the steal to still fail, even though there was always an item available. This is ok for the thread pool because it manages a known count of work items in the queues separately, and if it sees that there are still items available after a steal has failed, it'll try again. That "try again" logic provided above the work-stealing queue thus didn't make it over to ConcurrentBag, which breaks some usages of ConcurrentBag, in particular cases where a type like BlockingCollection is wrapping the bag and managing its own count. It's possible now for BlockingCollection to know that there's an item in the bag but to then fail to take it, which causes problems such as exceptions being thrown. The fix is to port back the relevant portion of ConcurrentBag from .NET Core 1.x / .NET Framework, where local push operations on a list track the number of times the list transitions from empty to non-empty. A steal operation then looks at those counts prior to doing the steal, and if the steal fails, it looks again after: if the count has increased, it retries. This unfortunately means that local pushes on small lists are now more expensive than in .NET Core 2.0/2.1, as if there are <= 2 items in the list, it takes the lock, but this seems unavoidable given the work-stealing design. --- .../Collections/Concurrent/ConcurrentBag.cs | 177 ++++++++++++------ .../tests/ConcurrentBagTests.cs | 56 ++++++ 2 files changed, 176 insertions(+), 57 deletions(-) diff --git a/src/System.Collections.Concurrent/src/System/Collections/Concurrent/ConcurrentBag.cs b/src/System.Collections.Concurrent/src/System/Collections/Concurrent/ConcurrentBag.cs index 4cc955c76cc0..6554f6133141 100644 --- a/src/System.Collections.Concurrent/src/System/Collections/Concurrent/ConcurrentBag.cs +++ b/src/System.Collections.Concurrent/src/System/Collections/Concurrent/ConcurrentBag.cs @@ -32,9 +32,11 @@ namespace System.Collections.Concurrent public class ConcurrentBag : IProducerConsumerCollection, IReadOnlyCollection { /// The per-bag, per-thread work-stealing queues. - private ThreadLocal _locals; + private readonly ThreadLocal _locals; /// The head work stealing queue in a linked list of queues. private volatile WorkStealingQueue _workStealingQueues; + /// Number of times any list transitions from empty to non-empty. + private long _emptyToNonEmptyListTransitionCount; /// Initializes a new instance of the class. public ConcurrentBag() @@ -62,7 +64,7 @@ public ConcurrentBag(IEnumerable collection) WorkStealingQueue queue = GetCurrentThreadWorkStealingQueue(forceCreate: true); foreach (T item in collection) { - queue.LocalPush(item); + queue.LocalPush(item, ref _emptyToNonEmptyListTransitionCount); } } @@ -72,7 +74,9 @@ public ConcurrentBag(IEnumerable collection) /// The object to be added to the /// . The value can be a null reference /// (Nothing in Visual Basic) for reference types. - public void Add(T item) => GetCurrentThreadWorkStealingQueue(forceCreate: true).LocalPush(item); + public void Add(T item) => + GetCurrentThreadWorkStealingQueue(forceCreate: true) + .LocalPush(item, ref _emptyToNonEmptyListTransitionCount); /// /// Attempts to add an object to the . @@ -176,22 +180,55 @@ private bool TrySteal(out T result, bool take) CDSCollectionETWBCLProvider.Log.ConcurrentBag_TryPeekSteals(); } - // If there's no local queue for this thread, just start from the head queue - // and try to steal from each queue until we get a result. - WorkStealingQueue localQueue = GetCurrentThreadWorkStealingQueue(forceCreate: false); - if (localQueue == null) + while (true) { - return TryStealFromTo(_workStealingQueues, null, out result, take); - } + // We need to track whether any lists transition from empty to non-empty both before + // and after we attempt the steal in case we don't get an item: + // + // If we don't get an item, we need to handle the possibility of a race condition that led to + // an item being added to a list after we already looked at it in a way that breaks + // linearizability. For example, say there are three threads 0, 1, and 2, each with their own + // list that's currently empty. We could then have the following series of operations: + // - Thread 2 adds an item, such that there's now 1 item in the bag. + // - Thread 1 sees that the count is 1 and does a Take. Its local list is empty, so it tries to + // steal from list 0, but it's empty. Before it can steal from Thread 2, it's pre-empted. + // - Thread 0 adds an item. The count is now 2. + // - Thread 2 takes an item, which comes from its local queue. The count is now 1. + // - Thread 1 continues to try to steal from 2, finds it's empty, and fails its take, even though + // at any given time during its take the count was >= 1. Oops. + // This is particularly problematic for wrapper types that track count using their own synchronization, + // e.g. BlockingCollection, and thus expect that a take will always be successful if the number of items + // is known to be > 0. + // + // We work around this by looking at the number of times any list transitions from == 0 to > 0, + // checking that before and after the steal attempts. We don't care about > 0 to > 0 transitions, + // because a steal from a list with > 0 elements would have been successful. + long initialEmptyToNonEmptyCounts = Interlocked.Read(ref _emptyToNonEmptyListTransitionCount); + + // If there's no local queue for this thread, just start from the head queue + // and try to steal from each queue until we get a result. If there is a local queue from this thread, + // then start from the next queue after it, and then iterate around back from the head to this queue, + // not including it. + WorkStealingQueue localQueue = GetCurrentThreadWorkStealingQueue(forceCreate: false); + bool gotItem = localQueue == null ? + TryStealFromTo(_workStealingQueues, null, out result, take) : + (TryStealFromTo(localQueue._nextQueue, null, out result, take) || TryStealFromTo(_workStealingQueues, localQueue, out result, take)); + if (gotItem) + { + return true; + } - // If there is a local queue from this thread, then start from the next queue - // after it, and then iterate around back from the head to this queue, not including it. - return - TryStealFromTo(localQueue._nextQueue, null, out result, take) || - TryStealFromTo(_workStealingQueues, localQueue, out result, take); + if (Interlocked.Read(ref _emptyToNonEmptyListTransitionCount) == initialEmptyToNonEmptyCounts) + { + // The version number matched, so we didn't get an item and we're confident enough + // in our steal attempt to say so. + return false; + } - // TODO: Investigate storing the queues in an array instead of a linked list, and then - // randomly choosing a starting location from which to start iterating. + // Some list transitioned from empty to non-empty between just before the steal and now. + // Since we don't know if it caused a race condition like the above description, we + // have little choice but to try to steal again. + } } /// @@ -684,7 +721,7 @@ internal bool IsEmpty /// Add new item to the tail of the queue. /// /// The item to add. - internal void LocalPush(T item) + internal void LocalPush(T item, ref long emptyToNonEmptyListTransitionCount) { Debug.Assert(Environment.CurrentManagedThreadId == _ownerThreadId); bool lockTaken = false; @@ -701,7 +738,7 @@ internal void LocalPush(T item) _currentOp = (int)Operation.None; // set back to None temporarily to avoid a deadlock lock (this) { - Debug.Assert(_tailIndex == int.MaxValue, "No other thread should be changing _tailIndex"); + Debug.Assert(_tailIndex == tail, "No other thread should be changing _tailIndex"); // Rather than resetting to zero, we'll just mask off the bits we don't care about. // This way we don't need to rearrange the items already in the queue; they'll be found @@ -711,22 +748,31 @@ internal void LocalPush(T item) // bits are set, so all of the bits we're keeping will also be set. Thus it's impossible // for the head to end up > than the tail, since you can't set any more bits than all of them. _headIndex = _headIndex & _mask; - _tailIndex = tail = _tailIndex & _mask; + _tailIndex = tail = tail & _mask; Debug.Assert(_headIndex <= _tailIndex); - _currentOp = (int)Operation.Add; + Interlocked.Exchange(ref _currentOp, (int)Operation.Add); // ensure subsequent reads aren't reordered before this } } - // We'd like to take the fast path that doesn't require locking, if possible. It's not possible if another - // thread is currently requesting that the whole bag synchronize, e.g. a ToArray operation. It's also - // not possible if there are fewer than two spaces available. One space is necessary for obvious reasons: - // to store the element we're trying to push. The other is necessary due to synchronization with steals. - // A stealing thread first increments _headIndex to reserve the slot at its old value, and then tries to - // read from that slot. We could potentially have a race condition whereby _headIndex is incremented just - // before this check, in which case we could overwrite the element being stolen as that slot would appear - // to be empty. Thus, we only allow the fast path if there are two empty slots. - if (!_frozen && tail < (_headIndex + _mask)) + // We'd like to take the fast path that doesn't require locking, if possible. It's not possible if: + // - another thread is currently requesting that the whole bag synchronize, e.g. a ToArray operation + // - if there are fewer than two spaces available. One space is necessary for obvious reasons: + // to store the element we're trying to push. The other is necessary due to synchronization with steals. + // A stealing thread first increments _headIndex to reserve the slot at its old value, and then tries to + // read from that slot. We could potentially have a race condition whereby _headIndex is incremented just + // before this check, in which case we could overwrite the element being stolen as that slot would appear + // to be empty. Thus, we only allow the fast path if there are two empty slots. + // - if there <= 1 elements in the list. We need to be able to successfully track transitions from + // empty to non-empty in a way that other threads can check, and we can only do that tracking + // correctly if we synchronize with steals when it's possible a steal could take the last item + // in the list just as we're adding. It's possible at this point that there's currently an active steal + // operation happening but that it hasn't yet incremented the head index, such that we could read a smaller + // than accurate by 1 value for the head. However, only one steal could possibly be doing so, as steals + // take the lock, and another steal couldn't then increment the header further because it'll see that + // there's currently an add operation in progress and wait until the add completes. + int head = _headIndex; // read after _currentOp set to Add + if (!_frozen && head < tail - 1 & tail < (head + _mask)) { _array[tail & _mask] = item; _tailIndex = tail + 1; @@ -737,8 +783,8 @@ internal void LocalPush(T item) _currentOp = (int)Operation.None; // set back to None to avoid a deadlock Monitor.Enter(this, ref lockTaken); - int head = _headIndex; - int count = _tailIndex - _headIndex; + head = _headIndex; + int count = tail - head; // this count is stable, as we're holding the lock // If we're full, expand the array. if (count >= _mask) @@ -767,6 +813,14 @@ internal void LocalPush(T item) _array[tail & _mask] = item; _tailIndex = tail + 1; + // Now that the item has been added, if we were at 0 (now at 1) item, + // increase the empty to non-empty transition count. + if (count == 0) + { + // We just transitioned from empty to non-empty, so increment the transition count. + Interlocked.Increment(ref emptyToNonEmptyListTransitionCount); + } + // Update the count to avoid overflow. We can trust _stealCount here, // as we're inside the lock and it's only manipulated there. _addTakeCount -= _stealCount; @@ -908,41 +962,50 @@ internal bool TryLocalPeek(out T result) /// true to take the item; false to simply peek at it internal bool TrySteal(out T result, bool take) { - // Fast-path check to see if the queue is empty. - if (_headIndex < _tailIndex) + lock (this) { - // Anything other than empty requires synchronization. - lock (this) + int head = _headIndex; // _headIndex is only manipulated under the lock + if (take) { - int head = _headIndex; - if (take) + // If there are <= 2 items in the list, we need to ensure no add operation + // is in progress, as add operations need to accurately count transitions + // from empty to non-empty, and they can only do that if there are no concurrent + // steal operations happening at the time. + if (head < _tailIndex - 1 && _currentOp != (int)Operation.Add) { - // Increment head to tentatively take an element: a full fence is used to ensure the read - // of _tailIndex doesn't move earlier, as otherwise we could potentially end up stealing - // the same element that's being popped locally. - Interlocked.Exchange(ref _headIndex, unchecked(head + 1)); - - // If there's an element to steal, do it. - if (head < _tailIndex) + var spinner = new SpinWait(); + do { - int idx = head & _mask; - result = _array[idx]; - _array[idx] = default(T); - _stealCount++; - return true; - } - else - { - // We contended with the local thread and lost the race, so restore the head. - _headIndex = head; + spinner.SpinOnce(); } + while (_currentOp == (int)Operation.Add); } - else if (head < _tailIndex) + + // Increment head to tentatively take an element: a full fence is used to ensure the read + // of _tailIndex doesn't move earlier, as otherwise we could potentially end up stealing + // the same element that's being popped locally. + Interlocked.Exchange(ref _headIndex, unchecked(head + 1)); + + // If there's an element to steal, do it. + if (head < _tailIndex) { - // Peek, if there's an element available - result = _array[head & _mask]; + int idx = head & _mask; + result = _array[idx]; + _array[idx] = default(T); + _stealCount++; return true; } + else + { + // We contended with the local thread and lost the race, so restore the head. + _headIndex = head; + } + } + else if (head < _tailIndex) + { + // Peek, if there's an element available + result = _array[head & _mask]; + return true; } } diff --git a/src/System.Collections.Concurrent/tests/ConcurrentBagTests.cs b/src/System.Collections.Concurrent/tests/ConcurrentBagTests.cs index c18ecc524a47..3bce75bf997c 100644 --- a/src/System.Collections.Concurrent/tests/ConcurrentBagTests.cs +++ b/src/System.Collections.Concurrent/tests/ConcurrentBagTests.cs @@ -92,6 +92,62 @@ public static void AddManyItems_ThenTakeOnDifferentThread_ItemsOutputInExpectedO }, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default).GetAwaiter().GetResult(); } + [Fact] + public static void SingleProducerAdding_MultiConsumerTaking_SemaphoreThrottling_AllTakesSucceed() + { + var bag = new ConcurrentBag(); + var s = new SemaphoreSlim(0); + CountdownEvent ce = null; + const int ItemCount = 200_000; + + int producerNextValue = 0; + Action producer = null; + producer = delegate + { + ThreadPool.QueueUserWorkItem(delegate + { + bag.Add(producerNextValue++); + s.Release(); + if (producerNextValue < ItemCount) + { + producer(); + } + else + { + ce.Signal(); + } + }); + }; + + int consumed = 0; + Action consumer = null; + consumer = delegate + { + ThreadPool.QueueUserWorkItem(delegate + { + if (s.Wait(0)) + { + Assert.True(bag.TryTake(out _), "There's an item available, but we couldn't take it."); + Interlocked.Increment(ref consumed); + } + else if (Volatile.Read(ref consumed) >= ItemCount) + { + ce.Signal(); + return; + } + + consumer(); + }); + }; + + // one producer, two consumers + ce = new CountdownEvent(3); + producer(); + consumer(); + consumer(); + ce.Wait(); + } + [Theory] [InlineData(0)] [InlineData(1)] From c78b98e65694610827ad83e0b19dc0059843cf92 Mon Sep 17 00:00:00 2001 From: Wes Haggard Date: Tue, 17 Jul 2018 16:29:32 -0700 Subject: [PATCH 17/18] Fix Sync targets to not actually cause a build to occur as well The TraversalBuildDependsOn gets other things added to it like BuildAllProjects which we don't want for the Sync target so factoring out the common pre-steps and sharing just those with sync and build. --- build.proj | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/build.proj b/build.proj index 3b71a9577403..95aec47afe06 100644 --- a/build.proj +++ b/build.proj @@ -29,9 +29,15 @@ - + GenerateConfigurationProperties; - CreateOrUpdateCurrentVersionFile; + CreateOrUpdateCurrentVersionFile + + + + + + $(PrebuildSetupTargets); $(TraversalBuildDependsOn); @@ -42,7 +48,7 @@ - + From 778a21f4ccbcf408cc72959c4f0a9512c47120dd Mon Sep 17 00:00:00 2001 From: Dan Moseley Date: Wed, 18 Jul 2018 13:13:47 -0700 Subject: [PATCH 18/18] Revert "Ensure ConcurrentBag's TryTake is linearizable (#30947) (#31009)" (#31132) This reverts commit b6be5d7cf0e05f103614338d9aca77639f0644dd. --- .../Collections/Concurrent/ConcurrentBag.cs | 177 ++++++------------ .../tests/ConcurrentBagTests.cs | 56 ------ 2 files changed, 57 insertions(+), 176 deletions(-) diff --git a/src/System.Collections.Concurrent/src/System/Collections/Concurrent/ConcurrentBag.cs b/src/System.Collections.Concurrent/src/System/Collections/Concurrent/ConcurrentBag.cs index 6554f6133141..4cc955c76cc0 100644 --- a/src/System.Collections.Concurrent/src/System/Collections/Concurrent/ConcurrentBag.cs +++ b/src/System.Collections.Concurrent/src/System/Collections/Concurrent/ConcurrentBag.cs @@ -32,11 +32,9 @@ namespace System.Collections.Concurrent public class ConcurrentBag : IProducerConsumerCollection, IReadOnlyCollection { /// The per-bag, per-thread work-stealing queues. - private readonly ThreadLocal _locals; + private ThreadLocal _locals; /// The head work stealing queue in a linked list of queues. private volatile WorkStealingQueue _workStealingQueues; - /// Number of times any list transitions from empty to non-empty. - private long _emptyToNonEmptyListTransitionCount; /// Initializes a new instance of the class. public ConcurrentBag() @@ -64,7 +62,7 @@ public ConcurrentBag(IEnumerable collection) WorkStealingQueue queue = GetCurrentThreadWorkStealingQueue(forceCreate: true); foreach (T item in collection) { - queue.LocalPush(item, ref _emptyToNonEmptyListTransitionCount); + queue.LocalPush(item); } } @@ -74,9 +72,7 @@ public ConcurrentBag(IEnumerable collection) /// The object to be added to the /// . The value can be a null reference /// (Nothing in Visual Basic) for reference types. - public void Add(T item) => - GetCurrentThreadWorkStealingQueue(forceCreate: true) - .LocalPush(item, ref _emptyToNonEmptyListTransitionCount); + public void Add(T item) => GetCurrentThreadWorkStealingQueue(forceCreate: true).LocalPush(item); /// /// Attempts to add an object to the . @@ -180,55 +176,22 @@ private bool TrySteal(out T result, bool take) CDSCollectionETWBCLProvider.Log.ConcurrentBag_TryPeekSteals(); } - while (true) + // If there's no local queue for this thread, just start from the head queue + // and try to steal from each queue until we get a result. + WorkStealingQueue localQueue = GetCurrentThreadWorkStealingQueue(forceCreate: false); + if (localQueue == null) { - // We need to track whether any lists transition from empty to non-empty both before - // and after we attempt the steal in case we don't get an item: - // - // If we don't get an item, we need to handle the possibility of a race condition that led to - // an item being added to a list after we already looked at it in a way that breaks - // linearizability. For example, say there are three threads 0, 1, and 2, each with their own - // list that's currently empty. We could then have the following series of operations: - // - Thread 2 adds an item, such that there's now 1 item in the bag. - // - Thread 1 sees that the count is 1 and does a Take. Its local list is empty, so it tries to - // steal from list 0, but it's empty. Before it can steal from Thread 2, it's pre-empted. - // - Thread 0 adds an item. The count is now 2. - // - Thread 2 takes an item, which comes from its local queue. The count is now 1. - // - Thread 1 continues to try to steal from 2, finds it's empty, and fails its take, even though - // at any given time during its take the count was >= 1. Oops. - // This is particularly problematic for wrapper types that track count using their own synchronization, - // e.g. BlockingCollection, and thus expect that a take will always be successful if the number of items - // is known to be > 0. - // - // We work around this by looking at the number of times any list transitions from == 0 to > 0, - // checking that before and after the steal attempts. We don't care about > 0 to > 0 transitions, - // because a steal from a list with > 0 elements would have been successful. - long initialEmptyToNonEmptyCounts = Interlocked.Read(ref _emptyToNonEmptyListTransitionCount); - - // If there's no local queue for this thread, just start from the head queue - // and try to steal from each queue until we get a result. If there is a local queue from this thread, - // then start from the next queue after it, and then iterate around back from the head to this queue, - // not including it. - WorkStealingQueue localQueue = GetCurrentThreadWorkStealingQueue(forceCreate: false); - bool gotItem = localQueue == null ? - TryStealFromTo(_workStealingQueues, null, out result, take) : - (TryStealFromTo(localQueue._nextQueue, null, out result, take) || TryStealFromTo(_workStealingQueues, localQueue, out result, take)); - if (gotItem) - { - return true; - } + return TryStealFromTo(_workStealingQueues, null, out result, take); + } - if (Interlocked.Read(ref _emptyToNonEmptyListTransitionCount) == initialEmptyToNonEmptyCounts) - { - // The version number matched, so we didn't get an item and we're confident enough - // in our steal attempt to say so. - return false; - } + // If there is a local queue from this thread, then start from the next queue + // after it, and then iterate around back from the head to this queue, not including it. + return + TryStealFromTo(localQueue._nextQueue, null, out result, take) || + TryStealFromTo(_workStealingQueues, localQueue, out result, take); - // Some list transitioned from empty to non-empty between just before the steal and now. - // Since we don't know if it caused a race condition like the above description, we - // have little choice but to try to steal again. - } + // TODO: Investigate storing the queues in an array instead of a linked list, and then + // randomly choosing a starting location from which to start iterating. } /// @@ -721,7 +684,7 @@ internal bool IsEmpty /// Add new item to the tail of the queue. /// /// The item to add. - internal void LocalPush(T item, ref long emptyToNonEmptyListTransitionCount) + internal void LocalPush(T item) { Debug.Assert(Environment.CurrentManagedThreadId == _ownerThreadId); bool lockTaken = false; @@ -738,7 +701,7 @@ internal void LocalPush(T item, ref long emptyToNonEmptyListTransitionCount) _currentOp = (int)Operation.None; // set back to None temporarily to avoid a deadlock lock (this) { - Debug.Assert(_tailIndex == tail, "No other thread should be changing _tailIndex"); + Debug.Assert(_tailIndex == int.MaxValue, "No other thread should be changing _tailIndex"); // Rather than resetting to zero, we'll just mask off the bits we don't care about. // This way we don't need to rearrange the items already in the queue; they'll be found @@ -748,31 +711,22 @@ internal void LocalPush(T item, ref long emptyToNonEmptyListTransitionCount) // bits are set, so all of the bits we're keeping will also be set. Thus it's impossible // for the head to end up > than the tail, since you can't set any more bits than all of them. _headIndex = _headIndex & _mask; - _tailIndex = tail = tail & _mask; + _tailIndex = tail = _tailIndex & _mask; Debug.Assert(_headIndex <= _tailIndex); - Interlocked.Exchange(ref _currentOp, (int)Operation.Add); // ensure subsequent reads aren't reordered before this + _currentOp = (int)Operation.Add; } } - // We'd like to take the fast path that doesn't require locking, if possible. It's not possible if: - // - another thread is currently requesting that the whole bag synchronize, e.g. a ToArray operation - // - if there are fewer than two spaces available. One space is necessary for obvious reasons: - // to store the element we're trying to push. The other is necessary due to synchronization with steals. - // A stealing thread first increments _headIndex to reserve the slot at its old value, and then tries to - // read from that slot. We could potentially have a race condition whereby _headIndex is incremented just - // before this check, in which case we could overwrite the element being stolen as that slot would appear - // to be empty. Thus, we only allow the fast path if there are two empty slots. - // - if there <= 1 elements in the list. We need to be able to successfully track transitions from - // empty to non-empty in a way that other threads can check, and we can only do that tracking - // correctly if we synchronize with steals when it's possible a steal could take the last item - // in the list just as we're adding. It's possible at this point that there's currently an active steal - // operation happening but that it hasn't yet incremented the head index, such that we could read a smaller - // than accurate by 1 value for the head. However, only one steal could possibly be doing so, as steals - // take the lock, and another steal couldn't then increment the header further because it'll see that - // there's currently an add operation in progress and wait until the add completes. - int head = _headIndex; // read after _currentOp set to Add - if (!_frozen && head < tail - 1 & tail < (head + _mask)) + // We'd like to take the fast path that doesn't require locking, if possible. It's not possible if another + // thread is currently requesting that the whole bag synchronize, e.g. a ToArray operation. It's also + // not possible if there are fewer than two spaces available. One space is necessary for obvious reasons: + // to store the element we're trying to push. The other is necessary due to synchronization with steals. + // A stealing thread first increments _headIndex to reserve the slot at its old value, and then tries to + // read from that slot. We could potentially have a race condition whereby _headIndex is incremented just + // before this check, in which case we could overwrite the element being stolen as that slot would appear + // to be empty. Thus, we only allow the fast path if there are two empty slots. + if (!_frozen && tail < (_headIndex + _mask)) { _array[tail & _mask] = item; _tailIndex = tail + 1; @@ -783,8 +737,8 @@ internal void LocalPush(T item, ref long emptyToNonEmptyListTransitionCount) _currentOp = (int)Operation.None; // set back to None to avoid a deadlock Monitor.Enter(this, ref lockTaken); - head = _headIndex; - int count = tail - head; // this count is stable, as we're holding the lock + int head = _headIndex; + int count = _tailIndex - _headIndex; // If we're full, expand the array. if (count >= _mask) @@ -813,14 +767,6 @@ internal void LocalPush(T item, ref long emptyToNonEmptyListTransitionCount) _array[tail & _mask] = item; _tailIndex = tail + 1; - // Now that the item has been added, if we were at 0 (now at 1) item, - // increase the empty to non-empty transition count. - if (count == 0) - { - // We just transitioned from empty to non-empty, so increment the transition count. - Interlocked.Increment(ref emptyToNonEmptyListTransitionCount); - } - // Update the count to avoid overflow. We can trust _stealCount here, // as we're inside the lock and it's only manipulated there. _addTakeCount -= _stealCount; @@ -962,50 +908,41 @@ internal bool TryLocalPeek(out T result) /// true to take the item; false to simply peek at it internal bool TrySteal(out T result, bool take) { - lock (this) + // Fast-path check to see if the queue is empty. + if (_headIndex < _tailIndex) { - int head = _headIndex; // _headIndex is only manipulated under the lock - if (take) + // Anything other than empty requires synchronization. + lock (this) { - // If there are <= 2 items in the list, we need to ensure no add operation - // is in progress, as add operations need to accurately count transitions - // from empty to non-empty, and they can only do that if there are no concurrent - // steal operations happening at the time. - if (head < _tailIndex - 1 && _currentOp != (int)Operation.Add) + int head = _headIndex; + if (take) { - var spinner = new SpinWait(); - do + // Increment head to tentatively take an element: a full fence is used to ensure the read + // of _tailIndex doesn't move earlier, as otherwise we could potentially end up stealing + // the same element that's being popped locally. + Interlocked.Exchange(ref _headIndex, unchecked(head + 1)); + + // If there's an element to steal, do it. + if (head < _tailIndex) { - spinner.SpinOnce(); + int idx = head & _mask; + result = _array[idx]; + _array[idx] = default(T); + _stealCount++; + return true; + } + else + { + // We contended with the local thread and lost the race, so restore the head. + _headIndex = head; } - while (_currentOp == (int)Operation.Add); } - - // Increment head to tentatively take an element: a full fence is used to ensure the read - // of _tailIndex doesn't move earlier, as otherwise we could potentially end up stealing - // the same element that's being popped locally. - Interlocked.Exchange(ref _headIndex, unchecked(head + 1)); - - // If there's an element to steal, do it. - if (head < _tailIndex) + else if (head < _tailIndex) { - int idx = head & _mask; - result = _array[idx]; - _array[idx] = default(T); - _stealCount++; + // Peek, if there's an element available + result = _array[head & _mask]; return true; } - else - { - // We contended with the local thread and lost the race, so restore the head. - _headIndex = head; - } - } - else if (head < _tailIndex) - { - // Peek, if there's an element available - result = _array[head & _mask]; - return true; } } diff --git a/src/System.Collections.Concurrent/tests/ConcurrentBagTests.cs b/src/System.Collections.Concurrent/tests/ConcurrentBagTests.cs index 3bce75bf997c..c18ecc524a47 100644 --- a/src/System.Collections.Concurrent/tests/ConcurrentBagTests.cs +++ b/src/System.Collections.Concurrent/tests/ConcurrentBagTests.cs @@ -92,62 +92,6 @@ public static void AddManyItems_ThenTakeOnDifferentThread_ItemsOutputInExpectedO }, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default).GetAwaiter().GetResult(); } - [Fact] - public static void SingleProducerAdding_MultiConsumerTaking_SemaphoreThrottling_AllTakesSucceed() - { - var bag = new ConcurrentBag(); - var s = new SemaphoreSlim(0); - CountdownEvent ce = null; - const int ItemCount = 200_000; - - int producerNextValue = 0; - Action producer = null; - producer = delegate - { - ThreadPool.QueueUserWorkItem(delegate - { - bag.Add(producerNextValue++); - s.Release(); - if (producerNextValue < ItemCount) - { - producer(); - } - else - { - ce.Signal(); - } - }); - }; - - int consumed = 0; - Action consumer = null; - consumer = delegate - { - ThreadPool.QueueUserWorkItem(delegate - { - if (s.Wait(0)) - { - Assert.True(bag.TryTake(out _), "There's an item available, but we couldn't take it."); - Interlocked.Increment(ref consumed); - } - else if (Volatile.Read(ref consumed) >= ItemCount) - { - ce.Signal(); - return; - } - - consumer(); - }); - }; - - // one producer, two consumers - ce = new CountdownEvent(3); - producer(); - consumer(); - consumer(); - ce.Wait(); - } - [Theory] [InlineData(0)] [InlineData(1)]