From 32121069814be8acc8e899fe1804d2ab4064d2f7 Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Sun, 26 Nov 2017 19:18:25 +0100 Subject: [PATCH 1/6] Reuse HashHelpers for BinaryFormatter objectholder hashes --- .../src/System/Collections/HashHelpers.cs | 5 - .../Data/SqlClient/SqlMetadataFactory.cs | 289 ------------------ .../src/Resources/Strings.resx | 5 +- ...em.Runtime.Serialization.Formatters.csproj | 5 +- .../Serialization/ObjectIDGenerator.cs | 31 +- 5 files changed, 19 insertions(+), 316 deletions(-) delete mode 100644 src/System.Data.SqlClient/src/System/Data/SqlClient/SqlMetadataFactory.cs diff --git a/src/Common/src/System/Collections/HashHelpers.cs b/src/Common/src/System/Collections/HashHelpers.cs index 661e9faf2e9e..de2789c3c2d3 100644 --- a/src/Common/src/System/Collections/HashHelpers.cs +++ b/src/Common/src/System/Collections/HashHelpers.cs @@ -12,12 +12,7 @@ ** ===========================================================*/ -using System; using System.Diagnostics; -using System.Runtime; -using System.Runtime.CompilerServices; -using System.Runtime.Serialization; -using System.Threading; namespace System.Collections { diff --git a/src/System.Data.SqlClient/src/System/Data/SqlClient/SqlMetadataFactory.cs b/src/System.Data.SqlClient/src/System/Data/SqlClient/SqlMetadataFactory.cs deleted file mode 100644 index 40b9b353c7bf..000000000000 --- a/src/System.Data.SqlClient/src/System/Data/SqlClient/SqlMetadataFactory.cs +++ /dev/null @@ -1,289 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Data; -using System.IO; -using System.Data.ProviderBase; -using System.Data.Common; -using System.Text; - -namespace System.Data.SqlClient -{ - internal sealed class SqlMetaDataFactory : DbMetaDataFactory - { - - private const string _serverVersionNormalized90 = "09.00.0000"; - private const string _serverVersionNormalized90782 = "09.00.0782"; - private const string _serverVersionNormalized10 = "10.00.0000"; - - - public SqlMetaDataFactory(Stream XMLStream, - string serverVersion, - string serverVersionNormalized) : - base(XMLStream, serverVersion, serverVersionNormalized) { } - - private void addUDTsToDataTypesTable(DataTable dataTypesTable, SqlConnection connection, String ServerVersion) - { - const string sqlCommand = - "select " + - "assemblies.name, " + - "types.assembly_class, " + - "ASSEMBLYPROPERTY(assemblies.name, 'VersionMajor') as version_major, " + - "ASSEMBLYPROPERTY(assemblies.name, 'VersionMinor') as version_minor, " + - "ASSEMBLYPROPERTY(assemblies.name, 'VersionBuild') as version_build, " + - "ASSEMBLYPROPERTY(assemblies.name, 'VersionRevision') as version_revision, " + - "ASSEMBLYPROPERTY(assemblies.name, 'CultureInfo') as culture_info, " + - "ASSEMBLYPROPERTY(assemblies.name, 'PublicKey') as public_key, " + - "is_nullable, " + - "is_fixed_length, " + - "max_length " + - "from sys.assemblies as assemblies join sys.assembly_types as types " + - "on assemblies.assembly_id = types.assembly_id "; - - // pre 9.0/Yukon servers do not have UDTs - if (0 > string.Compare(ServerVersion, _serverVersionNormalized90, StringComparison.OrdinalIgnoreCase)) - { - return; - } - - // Execute the SELECT statement - SqlCommand command = connection.CreateCommand(); - command.CommandText = sqlCommand; - DataRow newRow = null; - DataColumn providerDbtype = dataTypesTable.Columns[DbMetaDataColumnNames.ProviderDbType]; - DataColumn columnSize = dataTypesTable.Columns[DbMetaDataColumnNames.ColumnSize]; - DataColumn isFixedLength = dataTypesTable.Columns[DbMetaDataColumnNames.IsFixedLength]; - DataColumn isSearchable = dataTypesTable.Columns[DbMetaDataColumnNames.IsSearchable]; - DataColumn isLiteralSupported = dataTypesTable.Columns[DbMetaDataColumnNames.IsLiteralSupported]; - DataColumn typeName = dataTypesTable.Columns[DbMetaDataColumnNames.TypeName]; - DataColumn isNullable = dataTypesTable.Columns[DbMetaDataColumnNames.IsNullable]; - - if ((providerDbtype == null) || - (columnSize == null) || - (isFixedLength == null) || - (isSearchable == null) || - (isLiteralSupported == null) || - (typeName == null) || - (isNullable == null)) - { - throw ADP.InvalidXml(); - } - - const int columnSizeIndex = 10; - const int isFixedLengthIndex = 9; - const int isNullableIndex = 8; - const int assemblyNameIndex = 0; - const int assemblyClassIndex = 1; - const int versionMajorIndex = 2; - const int versionMinorIndex = 3; - const int versionBuildIndex = 4; - const int versionRevisionIndex = 5; - const int cultureInfoIndex = 6; - const int publicKeyIndex = 7; - - - using (IDataReader reader = command.ExecuteReader()) - { - - object[] values = new object[11]; - while (reader.Read()) - { - - reader.GetValues(values); - newRow = dataTypesTable.NewRow(); - - newRow[providerDbtype] = SqlDbType.Udt; - - if (values[columnSizeIndex] != DBNull.Value) - { - newRow[columnSize] = values[columnSizeIndex]; - } - - if (values[isFixedLengthIndex] != DBNull.Value) - { - newRow[isFixedLength] = values[isFixedLengthIndex]; - } - - newRow[isSearchable] = true; - newRow[isLiteralSupported] = false; - if (values[isNullableIndex] != DBNull.Value) - { - newRow[isNullable] = values[isNullableIndex]; - } - - if ((values[assemblyNameIndex] != DBNull.Value) && - (values[assemblyClassIndex] != DBNull.Value) && - (values[versionMajorIndex] != DBNull.Value) && - (values[versionMinorIndex] != DBNull.Value) && - (values[versionBuildIndex] != DBNull.Value) && - (values[versionRevisionIndex] != DBNull.Value)) - { - - StringBuilder nameString = new StringBuilder(); - nameString.Append(values[assemblyClassIndex].ToString()); - nameString.Append(", "); - nameString.Append(values[assemblyNameIndex].ToString()); - nameString.Append(", Version="); - - nameString.Append(values[versionMajorIndex].ToString()); - nameString.Append("."); - nameString.Append(values[versionMinorIndex].ToString()); - nameString.Append("."); - nameString.Append(values[versionBuildIndex].ToString()); - nameString.Append("."); - nameString.Append(values[versionRevisionIndex].ToString()); - - if (values[cultureInfoIndex] != DBNull.Value) - { - nameString.Append(", Culture="); - nameString.Append(values[cultureInfoIndex].ToString()); - } - - if (values[publicKeyIndex] != DBNull.Value) - { - - nameString.Append(", PublicKeyToken="); - - StringBuilder resultString = new StringBuilder(); - Byte[] byteArrayValue = (Byte[])values[publicKeyIndex]; - foreach (byte b in byteArrayValue) - { - resultString.Append(String.Format((IFormatProvider)null, "{0,-2:x2}", b)); - } - nameString.Append(resultString.ToString()); - } - - newRow[typeName] = nameString.ToString(); - dataTypesTable.Rows.Add(newRow); - newRow.AcceptChanges(); - } // if assembly name - - }//end while - } // end using - } - - private void AddTVPsToDataTypesTable(DataTable dataTypesTable, SqlConnection connection, String ServerVersion) - { - - const string sqlCommand = - "select " + - "name, " + - "is_nullable, " + - "max_length " + - "from sys.types " + - "where is_table_type = 1"; - - // TODO: update this check once the server upgrades major version number!!! - // pre 9.0/Yukon servers do not have Table types - if (0 > string.Compare(ServerVersion, _serverVersionNormalized10, StringComparison.OrdinalIgnoreCase)) - { - return; - } - - // Execute the SELECT statement - SqlCommand command = connection.CreateCommand(); - command.CommandText = sqlCommand; - DataRow newRow = null; - DataColumn providerDbtype = dataTypesTable.Columns[DbMetaDataColumnNames.ProviderDbType]; - DataColumn columnSize = dataTypesTable.Columns[DbMetaDataColumnNames.ColumnSize]; - DataColumn isSearchable = dataTypesTable.Columns[DbMetaDataColumnNames.IsSearchable]; - DataColumn isLiteralSupported = dataTypesTable.Columns[DbMetaDataColumnNames.IsLiteralSupported]; - DataColumn typeName = dataTypesTable.Columns[DbMetaDataColumnNames.TypeName]; - DataColumn isNullable = dataTypesTable.Columns[DbMetaDataColumnNames.IsNullable]; - - if ((providerDbtype == null) || - (columnSize == null) || - (isSearchable == null) || - (isLiteralSupported == null) || - (typeName == null) || - (isNullable == null)) - { - throw ADP.InvalidXml(); - } - - const int columnSizeIndex = 2; - const int isNullableIndex = 1; - const int typeNameIndex = 0; - - using (IDataReader reader = command.ExecuteReader()) - { - - object[] values = new object[11]; - while (reader.Read()) - { - - reader.GetValues(values); - newRow = dataTypesTable.NewRow(); - - newRow[providerDbtype] = SqlDbType.Structured; - - if (values[columnSizeIndex] != DBNull.Value) - { - newRow[columnSize] = values[columnSizeIndex]; - } - - newRow[isSearchable] = false; - newRow[isLiteralSupported] = false; - if (values[isNullableIndex] != DBNull.Value) - { - newRow[isNullable] = values[isNullableIndex]; - } - - if (values[typeNameIndex] != DBNull.Value) - { - newRow[typeName] = values[typeNameIndex]; - dataTypesTable.Rows.Add(newRow); - newRow.AcceptChanges(); - } // if type name - }//end while - } // end using - } - - private DataTable GetDataTypesTable(SqlConnection connection) - { - // verify the existance of the table in the data set - DataTable dataTypesTable = CollectionDataSet.Tables[DbMetaDataCollectionNames.DataTypes]; - if (dataTypesTable == null) - { - throw ADP.UnableToBuildCollection(DbMetaDataCollectionNames.DataTypes); - } - - // copy the table filtering out any rows that don't apply to tho current version of the prrovider - dataTypesTable = CloneAndFilterCollection(DbMetaDataCollectionNames.DataTypes, null); - - addUDTsToDataTypesTable(dataTypesTable, connection, ServerVersionNormalized); - AddTVPsToDataTypesTable(dataTypesTable, connection, ServerVersionNormalized); - - dataTypesTable.AcceptChanges(); - return dataTypesTable; - } - - protected override DataTable PrepareCollection(String collectionName, String[] restrictions, DbConnection connection) - { - SqlConnection sqlConnection = (SqlConnection)connection; - DataTable resultTable = null; - - if (collectionName == DbMetaDataCollectionNames.DataTypes) - { - if (ADP.IsEmptyArray(restrictions) == false) - { - throw ADP.TooManyRestrictions(DbMetaDataCollectionNames.DataTypes); - } - resultTable = GetDataTypesTable(sqlConnection); - } - - if (resultTable == null) - { - throw ADP.UnableToBuildCollection(collectionName); - } - - return resultTable; - - } - - - - } -} \ No newline at end of file diff --git a/src/System.Runtime.Serialization.Formatters/src/Resources/Strings.resx b/src/System.Runtime.Serialization.Formatters/src/Resources/Strings.resx index 8311d781049f..4783ed048c68 100644 --- a/src/System.Runtime.Serialization.Formatters/src/Resources/Strings.resx +++ b/src/System.Runtime.Serialization.Formatters/src/Resources/Strings.resx @@ -58,6 +58,9 @@ System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + Hashtable's capacity overflowed and went negative. Check load factor, capacity and the current size of the table. + Type '{0}' in Assembly '{1}' is not marked as serializable. @@ -247,4 +250,4 @@ Unable to read beyond the end of the stream. - \ No newline at end of file + diff --git a/src/System.Runtime.Serialization.Formatters/src/System.Runtime.Serialization.Formatters.csproj b/src/System.Runtime.Serialization.Formatters/src/System.Runtime.Serialization.Formatters.csproj index 7752ff98795d..ae432a86ca46 100644 --- a/src/System.Runtime.Serialization.Formatters/src/System.Runtime.Serialization.Formatters.csproj +++ b/src/System.Runtime.Serialization.Formatters/src/System.Runtime.Serialization.Formatters.csproj @@ -61,6 +61,9 @@ + + Common\System\Collections\HashHelpers.cs + @@ -77,4 +80,4 @@ - \ No newline at end of file + diff --git a/src/System.Runtime.Serialization.Formatters/src/System/Runtime/Serialization/ObjectIDGenerator.cs b/src/System.Runtime.Serialization.Formatters/src/System/Runtime/Serialization/ObjectIDGenerator.cs index e6b465333ef4..cc4a885a67d5 100644 --- a/src/System.Runtime.Serialization.Formatters/src/System/Runtime/Serialization/ObjectIDGenerator.cs +++ b/src/System.Runtime.Serialization.Formatters/src/System/Runtime/Serialization/ObjectIDGenerator.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System.Collections; using System.Runtime.CompilerServices; namespace System.Runtime.Serialization @@ -10,24 +11,16 @@ public class ObjectIDGenerator { private const int NumBins = 4; - // Table of prime numbers to use as hash table sizes. Each entry is the - // smallest prime number larger than twice the previous entry. - private static readonly int[] s_sizes = - { - 5, 11, 29, 47, 97, 197, 397, 797, 1597, 3203, 6421, 12853, 25717, 51437, - 102877, 205759, 411527, 823117, 1646237, 3292489, 6584983 - }; - internal int _currentCount; - internal int _currentSize; - internal long[] _ids; - internal object[] _objs; + private int _currentSize; + private long[] _ids; + private object[] _objs; // Constructs a new ObjectID generator, initializing all of the necessary variables. public ObjectIDGenerator() { _currentCount = 1; - _currentSize = s_sizes[0]; + _currentSize = HashHelpers.primes[0]; // Starting with 3 _ids = new long[_currentSize * NumBins]; _objs = new object[_currentSize * NumBins]; } @@ -106,13 +99,12 @@ public virtual long GetId(object obj, out bool firstTime) // we return that id, otherwise we return 0. public virtual long HasId(object obj, out bool firstTime) { - bool found; - if (obj == null) { throw new ArgumentNullException(nameof(obj)); } + bool found; int pos = FindElement(obj, out found); if (found) { @@ -129,14 +121,14 @@ public virtual long HasId(object obj, out bool firstTime) // the old arrays into the new ones. Expensive but necessary. private void Rehash() { - int i = 0; - for (int currSize = _currentSize; i < s_sizes.Length && s_sizes[i] <= currSize; i++) ; - if (i == s_sizes.Length) + int currSize = _currentSize; + int newSize = HashHelpers.ExpandPrime(currSize); + if (newSize == currSize) { // We just walked off the end of the array. throw new SerializationException(SR.Serialization_TooManyElements); } - _currentSize = s_sizes[i]; + _currentSize = newSize; long[] newIds = new long[_currentSize * NumBins]; object[] newObjs = new object[_currentSize * NumBins]; @@ -151,8 +143,7 @@ private void Rehash() { if (oldObjs[j] != null) { - bool found; - int pos = FindElement(oldObjs[j], out found); + int pos = FindElement(oldObjs[j], out _); _objs[pos] = oldObjs[j]; _ids[pos] = oldIds[j]; } From a3d2616060d4fdbdf5e3cf06249ef937776f1600 Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Mon, 27 Nov 2017 15:11:52 -0800 Subject: [PATCH 2/6] Revert "Merge pull request #6203 from SunnyWar/master" This reverts commit ddf8ca02958c6a78d58d641ba2e7e0bce585d572, reversing changes made to 0a0ea7fe57876ef47188dfe8bd76fae00e48f8e7. --- .../src/System/Collections/HashHelpers.cs | 48 +++++++++++-------- .../src/System/Collections/Generic/HashSet.cs | 2 + 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/src/Common/src/System/Collections/HashHelpers.cs b/src/Common/src/System/Collections/HashHelpers.cs index de2789c3c2d3..e5e0f4c0922a 100644 --- a/src/Common/src/System/Collections/HashHelpers.cs +++ b/src/Common/src/System/Collections/HashHelpers.cs @@ -2,22 +2,18 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -/*============================================================ -** -** -** -** -** Purpose: Hash table implementation -** -** -===========================================================*/ using System.Diagnostics; namespace System.Collections { internal static class HashHelpers - { + { + // This is the maximum prime smaller than Array.MaxArrayLength + public const int MaxPrimeArrayLength = 0x7FEFFFFD; + + private const int HashPrime = 101; + // Table of prime numbers to use as hash table sizes. // A typical resize algorithm would pick the smallest prime number in this array // that is larger than twice the previous capacity. @@ -34,11 +30,22 @@ internal static class HashHelpers 1103, 1327, 1597, 1931, 2333, 2801, 3371, 4049, 4861, 5839, 7013, 8419, 10103, 12143, 14591, 17519, 21023, 25229, 30293, 36353, 43627, 52361, 62851, 75431, 90523, 108631, 130363, 156437, 187751, 225307, 270371, 324449, 389357, 467237, 560689, 672827, 807403, 968897, 1162687, 1395263, - 1674319, 2009191, 2411033, 2893249, 3471899, 4166287, 4999559, 5999471, 7199369, 8639249, 10367101, - 12440537, 14928671, 17914409, 21497293, 25796759, 30956117, 37147349, 44576837, 53492207, 64190669, - 77028803, 92434613, 110921543, 133105859, 159727031, 191672443, 230006941, 276008387, 331210079, - 397452101, 476942527, 572331049, 686797261, 824156741, 988988137, 1186785773, 1424142949, 1708971541, - 2050765853, MaxPrimeArrayLength }; + 1674319, 2009191, 2411033, 2893249, 3471899, 4166287, 4999559, 5999471, 7199369 }; + + public static bool IsPrime(int candidate) + { + if ((candidate & 1) != 0) + { + int limit = (int)Math.Sqrt(candidate); + for (int divisor = 3; divisor <= limit; divisor += 2) + { + if ((candidate % divisor) == 0) + return false; + } + return true; + } + return (candidate == 2); + } public static int GetPrime(int min) { @@ -51,6 +58,13 @@ public static int GetPrime(int min) if (prime >= min) return prime; } + //outside of our predefined table. + //compute the hard way. + for (int i = (min | 1); i < int.MaxValue; i += 2) + { + if (IsPrime(i) && ((i - 1) % HashPrime != 0)) + return i; + } return min; } @@ -69,9 +83,5 @@ public static int ExpandPrime(int oldSize) return GetPrime(newSize); } - - - // This is the maximum prime smaller than Array.MaxArrayLength - public const int MaxPrimeArrayLength = 0x7FEFFFFD; } } diff --git a/src/System.Collections/src/System/Collections/Generic/HashSet.cs b/src/System.Collections/src/System/Collections/Generic/HashSet.cs index e13246b56257..87726c5e1a39 100644 --- a/src/System.Collections/src/System/Collections/Generic/HashSet.cs +++ b/src/System.Collections/src/System/Collections/Generic/HashSet.cs @@ -1162,6 +1162,8 @@ private void IncreaseCapacity() /// private void SetCapacity(int newSize) { + Debug.Assert(HashHelpers.IsPrime(newSize), "New size is not prime!"); + Debug.Assert(_buckets != null, "SetCapacity called on a set with no elements"); Slot[] newSlots = new Slot[newSize]; From 06daf0e8febf0f86cdb96707948d76a938220e14 Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Mon, 27 Nov 2017 15:12:39 -0800 Subject: [PATCH 3/6] Change resource string, make HashTable reuse existing HashHelper --- .../src/System/Collections/HashHelpers.cs | 2 +- .../Data/SqlClient/SqlMetadataFactory.cs | 289 ++++++++++++++++++ .../src/System.Runtime.Extensions.csproj | 5 +- .../src/System/Collections/Hashtable.cs | 89 +----- .../src/Resources/Strings.resx | 4 +- 5 files changed, 302 insertions(+), 87 deletions(-) create mode 100644 src/System.Data.SqlClient/src/System/Data/SqlClient/SqlMetadataFactory.cs diff --git a/src/Common/src/System/Collections/HashHelpers.cs b/src/Common/src/System/Collections/HashHelpers.cs index e5e0f4c0922a..14d07cd8daba 100644 --- a/src/Common/src/System/Collections/HashHelpers.cs +++ b/src/Common/src/System/Collections/HashHelpers.cs @@ -73,7 +73,7 @@ public static int ExpandPrime(int oldSize) { int newSize = 2 * oldSize; - // Allow the hashtables to grow to maximum possible size (~2G elements) before encoutering capacity overflow. + // Allow the hashtables to grow to maximum possible size (~2G elements) before encountering capacity overflow. // Note that this check works even when _items.Length overflowed thanks to the (uint) cast if ((uint)newSize > MaxPrimeArrayLength && MaxPrimeArrayLength > oldSize) { diff --git a/src/System.Data.SqlClient/src/System/Data/SqlClient/SqlMetadataFactory.cs b/src/System.Data.SqlClient/src/System/Data/SqlClient/SqlMetadataFactory.cs new file mode 100644 index 000000000000..40b9b353c7bf --- /dev/null +++ b/src/System.Data.SqlClient/src/System/Data/SqlClient/SqlMetadataFactory.cs @@ -0,0 +1,289 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Data; +using System.IO; +using System.Data.ProviderBase; +using System.Data.Common; +using System.Text; + +namespace System.Data.SqlClient +{ + internal sealed class SqlMetaDataFactory : DbMetaDataFactory + { + + private const string _serverVersionNormalized90 = "09.00.0000"; + private const string _serverVersionNormalized90782 = "09.00.0782"; + private const string _serverVersionNormalized10 = "10.00.0000"; + + + public SqlMetaDataFactory(Stream XMLStream, + string serverVersion, + string serverVersionNormalized) : + base(XMLStream, serverVersion, serverVersionNormalized) { } + + private void addUDTsToDataTypesTable(DataTable dataTypesTable, SqlConnection connection, String ServerVersion) + { + const string sqlCommand = + "select " + + "assemblies.name, " + + "types.assembly_class, " + + "ASSEMBLYPROPERTY(assemblies.name, 'VersionMajor') as version_major, " + + "ASSEMBLYPROPERTY(assemblies.name, 'VersionMinor') as version_minor, " + + "ASSEMBLYPROPERTY(assemblies.name, 'VersionBuild') as version_build, " + + "ASSEMBLYPROPERTY(assemblies.name, 'VersionRevision') as version_revision, " + + "ASSEMBLYPROPERTY(assemblies.name, 'CultureInfo') as culture_info, " + + "ASSEMBLYPROPERTY(assemblies.name, 'PublicKey') as public_key, " + + "is_nullable, " + + "is_fixed_length, " + + "max_length " + + "from sys.assemblies as assemblies join sys.assembly_types as types " + + "on assemblies.assembly_id = types.assembly_id "; + + // pre 9.0/Yukon servers do not have UDTs + if (0 > string.Compare(ServerVersion, _serverVersionNormalized90, StringComparison.OrdinalIgnoreCase)) + { + return; + } + + // Execute the SELECT statement + SqlCommand command = connection.CreateCommand(); + command.CommandText = sqlCommand; + DataRow newRow = null; + DataColumn providerDbtype = dataTypesTable.Columns[DbMetaDataColumnNames.ProviderDbType]; + DataColumn columnSize = dataTypesTable.Columns[DbMetaDataColumnNames.ColumnSize]; + DataColumn isFixedLength = dataTypesTable.Columns[DbMetaDataColumnNames.IsFixedLength]; + DataColumn isSearchable = dataTypesTable.Columns[DbMetaDataColumnNames.IsSearchable]; + DataColumn isLiteralSupported = dataTypesTable.Columns[DbMetaDataColumnNames.IsLiteralSupported]; + DataColumn typeName = dataTypesTable.Columns[DbMetaDataColumnNames.TypeName]; + DataColumn isNullable = dataTypesTable.Columns[DbMetaDataColumnNames.IsNullable]; + + if ((providerDbtype == null) || + (columnSize == null) || + (isFixedLength == null) || + (isSearchable == null) || + (isLiteralSupported == null) || + (typeName == null) || + (isNullable == null)) + { + throw ADP.InvalidXml(); + } + + const int columnSizeIndex = 10; + const int isFixedLengthIndex = 9; + const int isNullableIndex = 8; + const int assemblyNameIndex = 0; + const int assemblyClassIndex = 1; + const int versionMajorIndex = 2; + const int versionMinorIndex = 3; + const int versionBuildIndex = 4; + const int versionRevisionIndex = 5; + const int cultureInfoIndex = 6; + const int publicKeyIndex = 7; + + + using (IDataReader reader = command.ExecuteReader()) + { + + object[] values = new object[11]; + while (reader.Read()) + { + + reader.GetValues(values); + newRow = dataTypesTable.NewRow(); + + newRow[providerDbtype] = SqlDbType.Udt; + + if (values[columnSizeIndex] != DBNull.Value) + { + newRow[columnSize] = values[columnSizeIndex]; + } + + if (values[isFixedLengthIndex] != DBNull.Value) + { + newRow[isFixedLength] = values[isFixedLengthIndex]; + } + + newRow[isSearchable] = true; + newRow[isLiteralSupported] = false; + if (values[isNullableIndex] != DBNull.Value) + { + newRow[isNullable] = values[isNullableIndex]; + } + + if ((values[assemblyNameIndex] != DBNull.Value) && + (values[assemblyClassIndex] != DBNull.Value) && + (values[versionMajorIndex] != DBNull.Value) && + (values[versionMinorIndex] != DBNull.Value) && + (values[versionBuildIndex] != DBNull.Value) && + (values[versionRevisionIndex] != DBNull.Value)) + { + + StringBuilder nameString = new StringBuilder(); + nameString.Append(values[assemblyClassIndex].ToString()); + nameString.Append(", "); + nameString.Append(values[assemblyNameIndex].ToString()); + nameString.Append(", Version="); + + nameString.Append(values[versionMajorIndex].ToString()); + nameString.Append("."); + nameString.Append(values[versionMinorIndex].ToString()); + nameString.Append("."); + nameString.Append(values[versionBuildIndex].ToString()); + nameString.Append("."); + nameString.Append(values[versionRevisionIndex].ToString()); + + if (values[cultureInfoIndex] != DBNull.Value) + { + nameString.Append(", Culture="); + nameString.Append(values[cultureInfoIndex].ToString()); + } + + if (values[publicKeyIndex] != DBNull.Value) + { + + nameString.Append(", PublicKeyToken="); + + StringBuilder resultString = new StringBuilder(); + Byte[] byteArrayValue = (Byte[])values[publicKeyIndex]; + foreach (byte b in byteArrayValue) + { + resultString.Append(String.Format((IFormatProvider)null, "{0,-2:x2}", b)); + } + nameString.Append(resultString.ToString()); + } + + newRow[typeName] = nameString.ToString(); + dataTypesTable.Rows.Add(newRow); + newRow.AcceptChanges(); + } // if assembly name + + }//end while + } // end using + } + + private void AddTVPsToDataTypesTable(DataTable dataTypesTable, SqlConnection connection, String ServerVersion) + { + + const string sqlCommand = + "select " + + "name, " + + "is_nullable, " + + "max_length " + + "from sys.types " + + "where is_table_type = 1"; + + // TODO: update this check once the server upgrades major version number!!! + // pre 9.0/Yukon servers do not have Table types + if (0 > string.Compare(ServerVersion, _serverVersionNormalized10, StringComparison.OrdinalIgnoreCase)) + { + return; + } + + // Execute the SELECT statement + SqlCommand command = connection.CreateCommand(); + command.CommandText = sqlCommand; + DataRow newRow = null; + DataColumn providerDbtype = dataTypesTable.Columns[DbMetaDataColumnNames.ProviderDbType]; + DataColumn columnSize = dataTypesTable.Columns[DbMetaDataColumnNames.ColumnSize]; + DataColumn isSearchable = dataTypesTable.Columns[DbMetaDataColumnNames.IsSearchable]; + DataColumn isLiteralSupported = dataTypesTable.Columns[DbMetaDataColumnNames.IsLiteralSupported]; + DataColumn typeName = dataTypesTable.Columns[DbMetaDataColumnNames.TypeName]; + DataColumn isNullable = dataTypesTable.Columns[DbMetaDataColumnNames.IsNullable]; + + if ((providerDbtype == null) || + (columnSize == null) || + (isSearchable == null) || + (isLiteralSupported == null) || + (typeName == null) || + (isNullable == null)) + { + throw ADP.InvalidXml(); + } + + const int columnSizeIndex = 2; + const int isNullableIndex = 1; + const int typeNameIndex = 0; + + using (IDataReader reader = command.ExecuteReader()) + { + + object[] values = new object[11]; + while (reader.Read()) + { + + reader.GetValues(values); + newRow = dataTypesTable.NewRow(); + + newRow[providerDbtype] = SqlDbType.Structured; + + if (values[columnSizeIndex] != DBNull.Value) + { + newRow[columnSize] = values[columnSizeIndex]; + } + + newRow[isSearchable] = false; + newRow[isLiteralSupported] = false; + if (values[isNullableIndex] != DBNull.Value) + { + newRow[isNullable] = values[isNullableIndex]; + } + + if (values[typeNameIndex] != DBNull.Value) + { + newRow[typeName] = values[typeNameIndex]; + dataTypesTable.Rows.Add(newRow); + newRow.AcceptChanges(); + } // if type name + }//end while + } // end using + } + + private DataTable GetDataTypesTable(SqlConnection connection) + { + // verify the existance of the table in the data set + DataTable dataTypesTable = CollectionDataSet.Tables[DbMetaDataCollectionNames.DataTypes]; + if (dataTypesTable == null) + { + throw ADP.UnableToBuildCollection(DbMetaDataCollectionNames.DataTypes); + } + + // copy the table filtering out any rows that don't apply to tho current version of the prrovider + dataTypesTable = CloneAndFilterCollection(DbMetaDataCollectionNames.DataTypes, null); + + addUDTsToDataTypesTable(dataTypesTable, connection, ServerVersionNormalized); + AddTVPsToDataTypesTable(dataTypesTable, connection, ServerVersionNormalized); + + dataTypesTable.AcceptChanges(); + return dataTypesTable; + } + + protected override DataTable PrepareCollection(String collectionName, String[] restrictions, DbConnection connection) + { + SqlConnection sqlConnection = (SqlConnection)connection; + DataTable resultTable = null; + + if (collectionName == DbMetaDataCollectionNames.DataTypes) + { + if (ADP.IsEmptyArray(restrictions) == false) + { + throw ADP.TooManyRestrictions(DbMetaDataCollectionNames.DataTypes); + } + resultTable = GetDataTypesTable(sqlConnection); + } + + if (resultTable == null) + { + throw ADP.UnableToBuildCollection(collectionName); + } + + return resultTable; + + } + + + + } +} \ No newline at end of file diff --git a/src/System.Runtime.Extensions/src/System.Runtime.Extensions.csproj b/src/System.Runtime.Extensions/src/System.Runtime.Extensions.csproj index d79435f95300..3637bd70d9ac 100644 --- a/src/System.Runtime.Extensions/src/System.Runtime.Extensions.csproj +++ b/src/System.Runtime.Extensions/src/System.Runtime.Extensions.csproj @@ -83,6 +83,9 @@ CoreLib\System\Text\ValueStringBuilder.cs + + Common\System\Collections\HashHelpers.cs + @@ -278,4 +281,4 @@ - \ No newline at end of file + diff --git a/src/System.Runtime.Extensions/src/System/Collections/Hashtable.cs b/src/System.Runtime.Extensions/src/System/Collections/Hashtable.cs index a279aada8bb0..2709e1172ed4 100644 --- a/src/System.Runtime.Extensions/src/System/Collections/Hashtable.cs +++ b/src/System.Runtime.Extensions/src/System/Collections/Hashtable.cs @@ -153,6 +153,9 @@ private struct bucket private IEqualityComparer _keycomparer; private Object _syncRoot; + private static ConditionalWeakTable s_serializationInfoTable; + private static ConditionalWeakTable SerializationInfoTable => LazyInitializer.EnsureInitialized(ref s_serializationInfoTable); + [Obsolete("Please use EqualityComparer property.")] protected IHashCodeProvider hcp { @@ -380,7 +383,7 @@ protected Hashtable(SerializationInfo info, StreamingContext context) //We can't do anything with the keys and values until the entire graph has been deserialized //and we have a reasonable estimate that GetHashCode is not going to fail. For the time being, //we'll just cache this. The graph is not valid until OnDeserialization has been called. - HashHelpers.SerializationInfoTable.Add(this, info); + SerializationInfoTable.Add(this, info); } // ?InitHash? is basically an implementation of classic DoubleHashing (see http://en.wikipedia.org/wiki/Double_hashing) @@ -1172,7 +1175,7 @@ public virtual void OnDeserialization(Object sender) } SerializationInfo siInfo; - HashHelpers.SerializationInfoTable.TryGetValue(this, out siInfo); + SerializationInfoTable.TryGetValue(this, out siInfo); if (siInfo == null) { @@ -1254,7 +1257,7 @@ public virtual void OnDeserialization(Object sender) _version = siInfo.GetInt32(VersionName); - HashHelpers.SerializationInfoTable.Remove(this); + SerializationInfoTable.Remove(this); } // Implements a Collection for the keys of a hashtable. An instance of this @@ -1640,84 +1643,4 @@ public KeyValuePairs[] Items } } } - - internal static class HashHelpers - { - // Table of prime numbers to use as hash table sizes. - // A typical resize algorithm would pick the smallest prime number in this array - // that is larger than twice the previous capacity. - // Suppose our Hashtable currently has capacity x and enough elements are added - // such that a resize needs to occur. Resizing first computes 2x then finds the - // first prime in the table greater than 2x, i.e. if primes are ordered - // p_1, p_2, ..., p_i, ..., it finds p_n such that p_n-1 < 2x < p_n. - // Doubling is important for preserving the asymptotic complexity of the - // hashtable operations such as add. Having a prime guarantees that double - // hashing does not lead to infinite loops. IE, your hash function will be - // h1(key) + i*h2(key), 0 <= i < size. h2 and the size must be relatively prime. - public static readonly int[] primes = { - 3, 7, 11, 17, 23, 29, 37, 47, 59, 71, 89, 107, 131, 163, 197, 239, 293, 353, 431, 521, 631, 761, 919, - 1103, 1327, 1597, 1931, 2333, 2801, 3371, 4049, 4861, 5839, 7013, 8419, 10103, 12143, 14591, - 17519, 21023, 25229, 30293, 36353, 43627, 52361, 62851, 75431, 90523, 108631, 130363, 156437, - 187751, 225307, 270371, 324449, 389357, 467237, 560689, 672827, 807403, 968897, 1162687, 1395263, - 1674319, 2009191, 2411033, 2893249, 3471899, 4166287, 4999559, 5999471, 7199369}; - - public static bool IsPrime(int candidate) - { - if ((candidate & 1) != 0) - { - int limit = (int)Math.Sqrt(candidate); - for (int divisor = 3; divisor <= limit; divisor += 2) - { - if ((candidate % divisor) == 0) - return false; - } - return true; - } - return (candidate == 2); - } - - public static int GetPrime(int min) - { - if (min < 0) - throw new ArgumentException(SR.Arg_HTCapacityOverflow); - - for (int i = 0; i < primes.Length; i++) - { - int prime = primes[i]; - if (prime >= min) return prime; - } - - //outside of our predefined table. - //compute the hard way. - for (int i = (min | 1); i < Int32.MaxValue; i += 2) - { - if (IsPrime(i) && ((i - 1) % Hashtable.HashPrime != 0)) - return i; - } - return min; - } - - // Returns size of hashtable to grow to. - public static int ExpandPrime(int oldSize) - { - int newSize = 2 * oldSize; - - // Allow the hashtables to grow to maximum possible size (~2G elements) before encountering capacity overflow. - // Note that this check works even when _items.Length overflowed thanks to the (uint) cast - if ((uint)newSize > MaxPrimeArrayLength && MaxPrimeArrayLength > oldSize) - { - Debug.Assert(MaxPrimeArrayLength == GetPrime(MaxPrimeArrayLength), "Invalid MaxPrimeArrayLength"); - return MaxPrimeArrayLength; - } - - return GetPrime(newSize); - } - - - // This is the maximum prime smaller than Array.MaxArrayLength - public const int MaxPrimeArrayLength = 0x7FEFFFFD; - - private static ConditionalWeakTable s_serializationInfoTable; - public static ConditionalWeakTable SerializationInfoTable => LazyInitializer.EnsureInitialized(ref s_serializationInfoTable); - } } diff --git a/src/System.Runtime.Serialization.Formatters/src/Resources/Strings.resx b/src/System.Runtime.Serialization.Formatters/src/Resources/Strings.resx index 4783ed048c68..4a01b4fadcbb 100644 --- a/src/System.Runtime.Serialization.Formatters/src/Resources/Strings.resx +++ b/src/System.Runtime.Serialization.Formatters/src/Resources/Strings.resx @@ -59,7 +59,7 @@ System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - Hashtable's capacity overflowed and went negative. Check load factor, capacity and the current size of the table. + Capacity overflowed and went negative. Type '{0}' in Assembly '{1}' is not marked as serializable. @@ -250,4 +250,4 @@ Unable to read beyond the end of the stream. - + \ No newline at end of file From 7bba89bd6ec47248191122c232e9acc1cb0f5956 Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Wed, 28 Mar 2018 18:27:54 +0200 Subject: [PATCH 4/6] Add comment describing hash number growth --- src/Common/src/System/Collections/HashHelpers.cs | 2 ++ .../src/System/Collections/Generic/HashSet.cs | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Common/src/System/Collections/HashHelpers.cs b/src/Common/src/System/Collections/HashHelpers.cs index 14d07cd8daba..eae3ec4dd8ef 100644 --- a/src/Common/src/System/Collections/HashHelpers.cs +++ b/src/Common/src/System/Collections/HashHelpers.cs @@ -25,6 +25,8 @@ internal static class HashHelpers // hashtable operations such as add. Having a prime guarantees that double // hashing does not lead to infinite loops. IE, your hash function will be // h1(key) + i*h2(key), 0 <= i < size. h2 and the size must be relatively prime. + // We prefer the low computation costs of higher prime numbers over the increased + // memory allocation of a fixed prime number when i.e. right sizing a HashSet. public static readonly int[] primes = { 3, 7, 11, 17, 23, 29, 37, 47, 59, 71, 89, 107, 131, 163, 197, 239, 293, 353, 431, 521, 631, 761, 919, 1103, 1327, 1597, 1931, 2333, 2801, 3371, 4049, 4861, 5839, 7013, 8419, 10103, 12143, 14591, diff --git a/src/System.Collections/src/System/Collections/Generic/HashSet.cs b/src/System.Collections/src/System/Collections/Generic/HashSet.cs index 87726c5e1a39..06484e79a604 100644 --- a/src/System.Collections/src/System/Collections/Generic/HashSet.cs +++ b/src/System.Collections/src/System/Collections/Generic/HashSet.cs @@ -1163,7 +1163,6 @@ private void IncreaseCapacity() private void SetCapacity(int newSize) { Debug.Assert(HashHelpers.IsPrime(newSize), "New size is not prime!"); - Debug.Assert(_buckets != null, "SetCapacity called on a set with no elements"); Slot[] newSlots = new Slot[newSize]; From af85154aefcdde35a1e51fd3d205ff387cf31eeb Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Wed, 28 Mar 2018 19:24:56 +0200 Subject: [PATCH 5/6] Add hash number growth tests for BinaryFormatter & HashSet --- src/Common/src/System/Collections/HashHelpers.cs | 4 ++-- .../HashSet/HashSet.Generic.Tests.netcoreapp.cs | 12 ++++++++++++ .../tests/System.Collections.Tests.csproj | 2 +- .../tests/BinaryFormatterTests.cs | 16 ++++++++++++++++ 4 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/Common/src/System/Collections/HashHelpers.cs b/src/Common/src/System/Collections/HashHelpers.cs index eae3ec4dd8ef..68a738409907 100644 --- a/src/Common/src/System/Collections/HashHelpers.cs +++ b/src/Common/src/System/Collections/HashHelpers.cs @@ -8,7 +8,7 @@ namespace System.Collections { internal static class HashHelpers - { + { // This is the maximum prime smaller than Array.MaxArrayLength public const int MaxPrimeArrayLength = 0x7FEFFFFD; @@ -26,7 +26,7 @@ internal static class HashHelpers // hashing does not lead to infinite loops. IE, your hash function will be // h1(key) + i*h2(key), 0 <= i < size. h2 and the size must be relatively prime. // We prefer the low computation costs of higher prime numbers over the increased - // memory allocation of a fixed prime number when i.e. right sizing a HashSet. + // memory allocation of a fixed prime number i.e. when right sizing a HashSet. public static readonly int[] primes = { 3, 7, 11, 17, 23, 29, 37, 47, 59, 71, 89, 107, 131, 163, 197, 239, 293, 353, 431, 521, 631, 761, 919, 1103, 1327, 1597, 1931, 2333, 2801, 3371, 4049, 4861, 5839, 7013, 8419, 10103, 12143, 14591, diff --git a/src/System.Collections/tests/Generic/HashSet/HashSet.Generic.Tests.netcoreapp.cs b/src/System.Collections/tests/Generic/HashSet/HashSet.Generic.Tests.netcoreapp.cs index aff2d7c7e1a8..2f9226db0b21 100644 --- a/src/System.Collections/tests/Generic/HashSet/HashSet.Generic.Tests.netcoreapp.cs +++ b/src/System.Collections/tests/Generic/HashSet/HashSet.Generic.Tests.netcoreapp.cs @@ -30,6 +30,18 @@ public void HashSet_Generic_Constructor_int_AddUpToAndBeyondCapacity(int capacit Assert.Equal(capacity + 1, set.Count); } + [Fact] + public void HashSet_Generic_Constructor_Capacity_ToNextPrimeNumber() + { + // Highest pre-computed number + 1. + const int Capacity = 7199370; + var set = new HashSet(Capacity); + + // Assert that the HashTable's capacity is set to the descendant prime number of the given one. + const int NextPrime = 7199371; + Assert.Equal(NextPrime, set.EnsureCapacity(0)); + } + [Fact] public void HashSet_Generic_Constructor_int_Negative_ThrowsArgumentOutOfRangeException() { diff --git a/src/System.Collections/tests/System.Collections.Tests.csproj b/src/System.Collections/tests/System.Collections.Tests.csproj index e85d478938a4..40039d447f5d 100644 --- a/src/System.Collections/tests/System.Collections.Tests.csproj +++ b/src/System.Collections/tests/System.Collections.Tests.csproj @@ -74,7 +74,7 @@ Common\System\Collections\DictionaryExtensions.cs - System\Runtime\Serialization\Formatters\BinaryFormatterHelpers.cs + Common\System\Runtime\Serialization\Formatters\BinaryFormatterHelpers.cs diff --git a/src/System.Runtime.Serialization.Formatters/tests/BinaryFormatterTests.cs b/src/System.Runtime.Serialization.Formatters/tests/BinaryFormatterTests.cs index 077f99c3fe84..b163a912b5e1 100644 --- a/src/System.Runtime.Serialization.Formatters/tests/BinaryFormatterTests.cs +++ b/src/System.Runtime.Serialization.Formatters/tests/BinaryFormatterTests.cs @@ -18,6 +18,22 @@ namespace System.Runtime.Serialization.Formatters.Tests { public partial class BinaryFormatterTests : RemoteExecutorTestBase { + [Theory] + [InlineData(2 * 6_584_983 + 1)] // previous limit + 1 + [InlineData(2 * 7_199_369 + 1)] // last pre-computed prime number + 1 + public void SerializeHugeObjectGraphs(int limit) + { + var pointArr = Enumerable.Range(0, limit) + .Select(i => new Point(i, i + 1)) + .ToList(); + + // This should not throw a SerializationException as we removed the artifical limit in the ObjectIDGenerator. + // Instead of round tripping we only serialize to minimize test time. + // This will throw on .NET Framework as the artificial limit is still enabled. + AssertExtensions.ThrowsIf(PlatformDetection.IsFullFramework, + () => BinaryFormatterHelpers.ToByteArray(pointArr)); + } + [Theory] [MemberData(nameof(BasicObjectsRoundtrip_MemberData))] public void ValidateBasicObjectsRoundtrip(object obj, FormatterAssemblyStyle assemblyFormat, TypeFilterLevel filterLevel, FormatterTypeStyle typeFormat) From b7c32d26325e52cf7426144f65a93561adf9a129 Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Wed, 28 Mar 2018 21:46:25 +0200 Subject: [PATCH 6/6] Disable on x86 because of OOMs --- .../tests/BinaryFormatterTests.cs | 23 +++++++++++++------ ...time.Serialization.Formatters.Tests.csproj | 1 + 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/System.Runtime.Serialization.Formatters/tests/BinaryFormatterTests.cs b/src/System.Runtime.Serialization.Formatters/tests/BinaryFormatterTests.cs index b163a912b5e1..a6b814711ae1 100644 --- a/src/System.Runtime.Serialization.Formatters/tests/BinaryFormatterTests.cs +++ b/src/System.Runtime.Serialization.Formatters/tests/BinaryFormatterTests.cs @@ -18,20 +18,29 @@ namespace System.Runtime.Serialization.Formatters.Tests { public partial class BinaryFormatterTests : RemoteExecutorTestBase { - [Theory] - [InlineData(2 * 6_584_983 + 1)] // previous limit + 1 - [InlineData(2 * 7_199_369 + 1)] // last pre-computed prime number + 1 + private static unsafe bool Is64Bit => sizeof(void*) == 8; + + // On 32-bit we can't test these high inputs as they cause OutOfMemoryExceptions. + [ConditionalTheory(nameof(Is64Bit))] + [InlineData(2 * 6_584_983 - 2)] // previous limit + [InlineData(2 * 7_199_369 - 2)] // last pre-computed prime number public void SerializeHugeObjectGraphs(int limit) { - var pointArr = Enumerable.Range(0, limit) + Point[] pointArr = Enumerable.Range(0, limit) .Select(i => new Point(i, i + 1)) - .ToList(); + .ToArray(); // This should not throw a SerializationException as we removed the artifical limit in the ObjectIDGenerator. // Instead of round tripping we only serialize to minimize test time. // This will throw on .NET Framework as the artificial limit is still enabled. - AssertExtensions.ThrowsIf(PlatformDetection.IsFullFramework, - () => BinaryFormatterHelpers.ToByteArray(pointArr)); + var bf = new BinaryFormatter(); + AssertExtensions.ThrowsIf(PlatformDetection.IsFullFramework, () => + { + using (MemoryStream ms = new MemoryStream()) + { + bf.Serialize(ms, pointArr); + } + }); } [Theory] diff --git a/src/System.Runtime.Serialization.Formatters/tests/System.Runtime.Serialization.Formatters.Tests.csproj b/src/System.Runtime.Serialization.Formatters/tests/System.Runtime.Serialization.Formatters.Tests.csproj index 3660ed19c4b1..b51c89f48807 100644 --- a/src/System.Runtime.Serialization.Formatters/tests/System.Runtime.Serialization.Formatters.Tests.csproj +++ b/src/System.Runtime.Serialization.Formatters/tests/System.Runtime.Serialization.Formatters.Tests.csproj @@ -4,6 +4,7 @@ {13CE5E71-D373-4EA6-B3CB-166FF089A42A} true + true