From 99bf4efa0d5c0eddd38505ccbd41f13033e2c3ac Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Wed, 21 Aug 2024 19:34:56 +0530 Subject: [PATCH 1/4] Add streaming support for netcore and netfx --- .../src/Microsoft/Data/SqlClient/TdsParser.cs | 5 +- .../Microsoft/Data/SqlClient/SqlDataReader.cs | 2 +- .../src/Microsoft/Data/SqlClient/TdsParser.cs | 5 +- .../src/Microsoft/Data/SqlClient/SqlEnums.cs | 1 + ....Data.SqlClient.ManualTesting.Tests.csproj | 1 + .../SQL/JsonTest/JsonStreamTest.cs | 146 ++++++++++++++++++ 6 files changed, 155 insertions(+), 5 deletions(-) create mode 100644 src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs diff --git a/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParser.cs b/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParser.cs index 02b26a3a1b..b68ad0dcc6 100644 --- a/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParser.cs +++ b/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParser.cs @@ -11911,7 +11911,7 @@ private async Task WriteTextFeed(TextDataFeed feed, Encoding encoding, bool need encoding = encoding ?? TextDataFeed.DefaultEncoding; - using (ConstrainedTextWriter writer = new ConstrainedTextWriter(new StreamWriter(new TdsOutputStream(this, stateObj, null), encoding), size)) + using (ConstrainedTextWriter writer = encoding == Encoding.UTF8 ? new ConstrainedTextWriter(new StreamWriter(new TdsOutputStream(this, stateObj, null)), size) : new ConstrainedTextWriter(new StreamWriter(new TdsOutputStream(this, stateObj, null), encoding), size)) { if (needBom) { @@ -12175,7 +12175,8 @@ private Task WriteUnterminatedValue(object value, MetaType type, byte scale, int } else { - return NullIfCompletedWriteTask(WriteTextFeed(tdf, null, IsBOMNeeded(type, value), stateObj, paramSize)); + Encoding encoding = type.NullableType == TdsEnums.SQLJSON ? Encoding.UTF8 : null; + return NullIfCompletedWriteTask(WriteTextFeed(tdf, encoding, IsBOMNeeded(type, value), stateObj, paramSize)); } } else diff --git a/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/SqlDataReader.cs b/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/SqlDataReader.cs index 0c0b15a330..9d2110181d 100644 --- a/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/SqlDataReader.cs +++ b/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/SqlDataReader.cs @@ -2283,7 +2283,7 @@ override public TextReader GetTextReader(int i) { encoding = _metaData[i].encoding; } - + encoding = mt.SqlDbType == SqlDbTypeExtensions.Json ? System.Text.Encoding.UTF8 : encoding; _currentTextReader = new SqlSequentialTextReader(this, i, encoding); _lastColumnWithDataChunkRead = i; return _currentTextReader; diff --git a/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/TdsParser.cs b/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/TdsParser.cs index 018078a66f..42e4c628a1 100644 --- a/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/TdsParser.cs +++ b/src/Microsoft.Data.SqlClient/netfx/src/Microsoft/Data/SqlClient/TdsParser.cs @@ -12904,7 +12904,7 @@ private async Task WriteTextFeed(TextDataFeed feed, Encoding encoding, bool need char[] inBuff = new char[constTextBufferSize]; encoding = encoding ?? new UnicodeEncoding(false, false); - ConstrainedTextWriter writer = new ConstrainedTextWriter(new StreamWriter(new TdsOutputStream(this, stateObj, null), encoding), size); + ConstrainedTextWriter writer = encoding == Encoding.UTF8 ? new ConstrainedTextWriter(new StreamWriter(new TdsOutputStream(this, stateObj, null)), size) : new ConstrainedTextWriter(new StreamWriter(new TdsOutputStream(this, stateObj, null), encoding), size); if (needBom) { @@ -13163,7 +13163,8 @@ private Task WriteUnterminatedValue(object value, MetaType type, byte scale, int } else { - return NullIfCompletedWriteTask(WriteTextFeed(tdf, null, IsBOMNeeded(type, value), stateObj, paramSize)); + Encoding encoding = type.NullableType == TdsEnums.SQLJSON ? Encoding.UTF8 : null; + return NullIfCompletedWriteTask(WriteTextFeed(tdf, encoding, IsBOMNeeded(type, value), stateObj, paramSize)); } } else diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlEnums.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlEnums.cs index dafb0c5ef5..758f7fd414 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlEnums.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlEnums.cs @@ -124,6 +124,7 @@ private static bool _IsCharType(SqlDbType type) => type == SqlDbType.Char || type == SqlDbType.VarChar || type == SqlDbType.Text || + type == SqlDbTypeExtensions.Json || type == SqlDbType.Xml; private static bool _IsNCharType(SqlDbType type) => diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj b/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj index 25baa19768..30e5e18d96 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj @@ -291,6 +291,7 @@ + diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs new file mode 100644 index 0000000000..5d69ae9ca5 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs @@ -0,0 +1,146 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Data; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Newtonsoft.Json; +using Xunit; +using Xunit.Abstractions; +using Newtonsoft.Json.Linq; + + +namespace Microsoft.Data.SqlClient.ManualTesting.Tests +{ + public class Record + { + public int Id { get; set; } + public string Name { get; set; } + } + + public class JsonStreamTest + { + private readonly ITestOutputHelper _output; + private static readonly string jsonFilename = "randomRecords.json"; + private static readonly string outputFilename = "serverRecords.json"; + + public JsonStreamTest(ITestOutputHelper output) + { + _output = output; + } + + private void generateJsonFile(int noOfRecords, string filename) + { + deleteFile(filename); + var random = new Random(); + var records = new List(); + int recordCount = noOfRecords; + + for (int i = 0; i < recordCount; i++) + { + records.Add(new Record + { + Id = i + 1, + Name = "json" + random.Next(1, noOfRecords), + }); + } + + string json = JsonConvert.SerializeObject(records, Formatting.Indented); + File.WriteAllText(filename, json); + Assert.True(File.Exists(filename)); + _output.WriteLine("Generated Json File "+filename); + } + + private void compareJsonFiles() + { + using (var stream1 = File.OpenText(jsonFilename)) + using (var stream2 = File.OpenText(outputFilename)) + using (var reader1 = new JsonTextReader(stream1)) + using (var reader2 = new JsonTextReader(stream2)) + { + var jToken1 = JToken.ReadFrom(reader1); + var jToken2 = JToken.ReadFrom(reader2); + Assert.True(JToken.DeepEquals(jToken1, jToken2)); + } + } + + private void printJsonDataToFile(SqlConnection connection) + { + deleteFile(outputFilename); + Console.OutputEncoding = Encoding.UTF8; + using (SqlCommand command = new SqlCommand("SELECT [data] FROM [jsonTab]", connection)) + { + using (SqlDataReader reader = command.ExecuteReader(CommandBehavior.SequentialAccess)) + { + using (StreamWriter sw = new StreamWriter(outputFilename)) + { + while (reader.Read()) + { + char[] buffer = new char[4096]; + int charsRead = 0; + + using (TextReader data = reader.GetTextReader(0)) + { + do + { + charsRead = data.Read(buffer, 0, buffer.Length); + sw.Write(buffer, 0, charsRead); + + } while (charsRead > 0); + } + _output.WriteLine("Output written to " + outputFilename); + } + } + } + } + } + + private void streamJsonFileToServer(SqlConnection connection) + { + using (SqlCommand cmd = new SqlCommand("INSERT INTO [jsonTab] (data) VALUES (@jsondata)", connection)) + { + using (StreamReader jsonFile = File.OpenText(jsonFilename)) + { + cmd.Parameters.Add("@jsondata", Microsoft.Data.SqlDbTypeExtensions.Json, -1).Value = jsonFile; + cmd.ExecuteNonQuery(); + } + } + } + + private void deleteFile(string filename) + { + if (File.Exists(filename)) + { + File.Delete(filename); + } + + } + + + [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.IsJsonSupported))] + public void TestJsonStreaming() + { + generateJsonFile(10000, jsonFilename); + + string tableCreate = "CREATE TABLE jsonTab(data json)"; + using (SqlConnection connection = new SqlConnection(DataTestUtility.TCPConnectionString)) + { + connection.Open(); + DataTestUtility.DropTable(connection, "jsonTab"); + using (SqlCommand command = connection.CreateCommand()) + { + //Create Table + command.CommandText = tableCreate; + command.ExecuteNonQuery(); + } + streamJsonFileToServer(connection); + printJsonDataToFile(connection); + compareJsonFiles(); + deleteFile(jsonFilename); + deleteFile(outputFilename); + } + } + } +} + From dab5aee357136e61095d35b991d91bf9dc91f1cd Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Wed, 21 Aug 2024 20:29:47 +0530 Subject: [PATCH 2/4] Add multibyte character for testing and use UTF8 encoding in netcore for GetTextReader --- .../netcore/src/Microsoft/Data/SqlClient/SqlDataReader.cs | 2 +- .../tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/SqlDataReader.cs b/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/SqlDataReader.cs index 70c91bc184..71d94375bb 100644 --- a/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/SqlDataReader.cs +++ b/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/SqlDataReader.cs @@ -1994,7 +1994,7 @@ override public TextReader GetTextReader(int i) { encoding = _metaData[i].encoding; } - + encoding = mt.SqlDbType == SqlDbTypeExtensions.Json ? System.Text.Encoding.UTF8 : encoding; _currentTextReader = new SqlSequentialTextReader(this, i, encoding); _lastColumnWithDataChunkRead = i; return _currentTextReader; diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs index 5d69ae9ca5..af50df9f0d 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs @@ -42,7 +42,7 @@ private void generateJsonFile(int noOfRecords, string filename) records.Add(new Record { Id = i + 1, - Name = "json" + random.Next(1, noOfRecords), + Name = "𩸽json" + random.Next(1, noOfRecords), }); } From aa956b5b575aff9d2859807ebd8363e93182e3c4 Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Mon, 26 Aug 2024 21:32:39 +0530 Subject: [PATCH 3/4] Add async testcase for streaming support --- .../ManualTests/DataCommon/DataTestUtility.cs | 11 ++ .../SQL/JsonTest/JsonStreamTest.cs | 117 +++++++++++++----- 2 files changed, 96 insertions(+), 32 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/DataCommon/DataTestUtility.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/DataCommon/DataTestUtility.cs index febde5acbc..3c12e15db8 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/DataCommon/DataTestUtility.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/DataCommon/DataTestUtility.cs @@ -590,6 +590,17 @@ public static string GetUniqueNameForSqlServer(string prefix, bool withBracket = return name; } + public static void CreateTable(SqlConnection sqlConnection, string tableName, string createBody) + { + DropTable(sqlConnection, tableName); + string tableCreate = "CREATE TABLE " + tableName + createBody; + using (SqlCommand command = sqlConnection.CreateCommand()) + { + command.CommandText = tableCreate; + command.ExecuteNonQuery(); + } + } + public static void DropTable(SqlConnection sqlConnection, string tableName) { ResurrectConnection(sqlConnection); diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs index af50df9f0d..80c6de7850 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs @@ -13,7 +13,7 @@ namespace Microsoft.Data.SqlClient.ManualTesting.Tests { - public class Record + public class JsonRecord { public int Id { get; set; } public string Name { get; set; } @@ -22,24 +22,24 @@ public class Record public class JsonStreamTest { private readonly ITestOutputHelper _output; - private static readonly string jsonFilename = "randomRecords.json"; - private static readonly string outputFilename = "serverRecords.json"; + private static readonly string _jsonFile = "randomRecords.json"; + private static readonly string _outputFile = "serverRecords.json"; public JsonStreamTest(ITestOutputHelper output) { _output = output; } - private void generateJsonFile(int noOfRecords, string filename) + private void GenerateJsonFile(int noOfRecords, string filename) { - deleteFile(filename); + DeleteFile(filename); var random = new Random(); - var records = new List(); + var records = new List(); int recordCount = noOfRecords; for (int i = 0; i < recordCount; i++) { - records.Add(new Record + records.Add(new JsonRecord { Id = i + 1, Name = "𩸽json" + random.Next(1, noOfRecords), @@ -49,13 +49,13 @@ private void generateJsonFile(int noOfRecords, string filename) string json = JsonConvert.SerializeObject(records, Formatting.Indented); File.WriteAllText(filename, json); Assert.True(File.Exists(filename)); - _output.WriteLine("Generated Json File "+filename); + _output.WriteLine("Generated JSON file "+filename); } - private void compareJsonFiles() + private void CompareJsonFiles() { - using (var stream1 = File.OpenText(jsonFilename)) - using (var stream2 = File.OpenText(outputFilename)) + using (var stream1 = File.OpenText(_jsonFile)) + using (var stream2 = File.OpenText(_outputFile)) using (var reader1 = new JsonTextReader(stream1)) using (var reader2 = new JsonTextReader(stream2)) { @@ -65,15 +65,15 @@ private void compareJsonFiles() } } - private void printJsonDataToFile(SqlConnection connection) + private void PrintJsonDataToFile(SqlConnection connection) { - deleteFile(outputFilename); + DeleteFile(_outputFile); Console.OutputEncoding = Encoding.UTF8; using (SqlCommand command = new SqlCommand("SELECT [data] FROM [jsonTab]", connection)) { using (SqlDataReader reader = command.ExecuteReader(CommandBehavior.SequentialAccess)) { - using (StreamWriter sw = new StreamWriter(outputFilename)) + using (StreamWriter sw = new StreamWriter(_outputFile)) { while (reader.Read()) { @@ -89,18 +89,49 @@ private void printJsonDataToFile(SqlConnection connection) } while (charsRead > 0); } - _output.WriteLine("Output written to " + outputFilename); + _output.WriteLine("Output written to " + _outputFile); } } } } } - private void streamJsonFileToServer(SqlConnection connection) + private async Task PrintJsonDataToFileAsync(SqlConnection connection) + { + DeleteFile(_outputFile); + Console.OutputEncoding = Encoding.UTF8; + using (SqlCommand command = new SqlCommand("SELECT [data] FROM [jsonTab]", connection)) + { + using (SqlDataReader reader = await command.ExecuteReaderAsync(CommandBehavior.SequentialAccess)) + { + using (StreamWriter sw = new StreamWriter(_outputFile)) + { + while (await reader.ReadAsync()) + { + char[] buffer = new char[4096]; + int charsRead = 0; + + using (TextReader data = reader.GetTextReader(0)) + { + do + { + charsRead = await data.ReadAsync(buffer, 0, buffer.Length); + await sw.WriteAsync(buffer, 0, charsRead); + + } while (charsRead > 0); + } + _output.WriteLine("Output written to file " + _outputFile); + } + } + } + } + } + + private void StreamJsonFileToServer(SqlConnection connection) { using (SqlCommand cmd = new SqlCommand("INSERT INTO [jsonTab] (data) VALUES (@jsondata)", connection)) { - using (StreamReader jsonFile = File.OpenText(jsonFilename)) + using (StreamReader jsonFile = File.OpenText(_jsonFile)) { cmd.Parameters.Add("@jsondata", Microsoft.Data.SqlDbTypeExtensions.Json, -1).Value = jsonFile; cmd.ExecuteNonQuery(); @@ -108,7 +139,19 @@ private void streamJsonFileToServer(SqlConnection connection) } } - private void deleteFile(string filename) + private async Task StreamJsonFileToServerAsync(SqlConnection connection) + { + using (SqlCommand cmd = new SqlCommand("INSERT INTO [jsonTab] (data) VALUES (@jsondata)", connection)) + { + using (StreamReader jsonFile = File.OpenText(_jsonFile)) + { + cmd.Parameters.Add("@jsondata", Microsoft.Data.SqlDbTypeExtensions.Json, -1).Value = jsonFile; + await cmd.ExecuteNonQueryAsync(); + } + } + } + + private void DeleteFile(string filename) { if (File.Exists(filename)) { @@ -121,24 +164,34 @@ private void deleteFile(string filename) [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.IsJsonSupported))] public void TestJsonStreaming() { - generateJsonFile(10000, jsonFilename); + GenerateJsonFile(10000, _jsonFile); - string tableCreate = "CREATE TABLE jsonTab(data json)"; using (SqlConnection connection = new SqlConnection(DataTestUtility.TCPConnectionString)) { connection.Open(); - DataTestUtility.DropTable(connection, "jsonTab"); - using (SqlCommand command = connection.CreateCommand()) - { - //Create Table - command.CommandText = tableCreate; - command.ExecuteNonQuery(); - } - streamJsonFileToServer(connection); - printJsonDataToFile(connection); - compareJsonFiles(); - deleteFile(jsonFilename); - deleteFile(outputFilename); + DataTestUtility.CreateTable(connection, "jsonTab", "(data json)"); + StreamJsonFileToServer(connection); + PrintJsonDataToFile(connection); + CompareJsonFiles(); + DeleteFile(_jsonFile); + DeleteFile(_outputFile); + } + } + + [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.IsJsonSupported))] + public async Task TestJsonStreamingAsync() + { + GenerateJsonFile(10000, _jsonFile); + + using (SqlConnection connection = new SqlConnection(DataTestUtility.TCPConnectionString)) + { + await connection.OpenAsync(); + DataTestUtility.CreateTable(connection, "jsonTab", "(data json)"); + await StreamJsonFileToServerAsync(connection); + await PrintJsonDataToFileAsync(connection); + CompareJsonFiles(); + DeleteFile(_jsonFile); + DeleteFile(_outputFile); } } } From c56da7bc5b70cf33bd0682b858cf32b89a03b830 Mon Sep 17 00:00:00 2001 From: Apoorv Deshmukh Date: Wed, 4 Sep 2024 00:35:18 +0530 Subject: [PATCH 4/4] Remove unnecessary UTF8 setting for Console --- .../tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs index 80c6de7850..e6c35c9b6e 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs @@ -68,7 +68,6 @@ private void CompareJsonFiles() private void PrintJsonDataToFile(SqlConnection connection) { DeleteFile(_outputFile); - Console.OutputEncoding = Encoding.UTF8; using (SqlCommand command = new SqlCommand("SELECT [data] FROM [jsonTab]", connection)) { using (SqlDataReader reader = command.ExecuteReader(CommandBehavior.SequentialAccess)) @@ -99,7 +98,6 @@ private void PrintJsonDataToFile(SqlConnection connection) private async Task PrintJsonDataToFileAsync(SqlConnection connection) { DeleteFile(_outputFile); - Console.OutputEncoding = Encoding.UTF8; using (SqlCommand command = new SqlCommand("SELECT [data] FROM [jsonTab]", connection)) { using (SqlDataReader reader = await command.ExecuteReaderAsync(CommandBehavior.SequentialAccess))