From bfb4eb33701299af50d03b575c697477b3546061 Mon Sep 17 00:00:00 2001 From: pumpkin-bit Date: Sat, 18 Apr 2026 12:12:08 +0500 Subject: [PATCH 1/9] Ensure buffer space before reading in Decompress method --- .../Zstandard/ZstandardStream.Decompress.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardStream.Decompress.cs b/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardStream.Decompress.cs index 4e24909d48af77..1324e9328ecef0 100644 --- a/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardStream.Decompress.cs +++ b/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardStream.Decompress.cs @@ -117,6 +117,11 @@ public override int Read(Span buffer) OperationStatus lastResult; while (!TryDecompress(buffer, out bytesWritten, out lastResult)) { + if (_buffer.AvailableLength == 0) + { + _buffer.EnsureAvailableSpace(1); + } + int bytesRead = _stream.Read(_buffer.AvailableSpan); if (bytesRead <= 0) { @@ -192,6 +197,11 @@ public override async ValueTask ReadAsync(Memory buffer, Cancellation OperationStatus lastResult; while (!TryDecompress(buffer.Span, out bytesWritten, out lastResult)) { + if (_buffer.AvailableLength == 0) + { + _buffer.EnsureAvailableSpace(1); + } + int bytesRead = await _stream.ReadAsync(_buffer.AvailableMemory, cancellationToken).ConfigureAwait(false); if (bytesRead <= 0) { From 693d56c247a968840669712768899750cc1102e8 Mon Sep 17 00:00:00 2001 From: pumpkin-bit Date: Sat, 18 Apr 2026 13:27:06 +0500 Subject: [PATCH 2/9] style: remove trailing whitespaces --- .../IO/Compression/Zstandard/ZstandardStream.Decompress.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardStream.Decompress.cs b/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardStream.Decompress.cs index 1324e9328ecef0..2ae591553a98f5 100644 --- a/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardStream.Decompress.cs +++ b/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardStream.Decompress.cs @@ -121,7 +121,6 @@ public override int Read(Span buffer) { _buffer.EnsureAvailableSpace(1); } - int bytesRead = _stream.Read(_buffer.AvailableSpan); if (bytesRead <= 0) { @@ -201,7 +200,6 @@ public override async ValueTask ReadAsync(Memory buffer, Cancellation { _buffer.EnsureAvailableSpace(1); } - int bytesRead = await _stream.ReadAsync(_buffer.AvailableMemory, cancellationToken).ConfigureAwait(false); if (bytesRead <= 0) { From 516c17870aec531cc5765a48fbeb69316468d48e Mon Sep 17 00:00:00 2001 From: pumpkin-bit Date: Sat, 18 Apr 2026 19:21:50 +0500 Subject: [PATCH 3/9] Update src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardStream.Decompress.cs Co-authored-by: Miha Zupan --- .../IO/Compression/Zstandard/ZstandardStream.Decompress.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardStream.Decompress.cs b/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardStream.Decompress.cs index 2ae591553a98f5..8e1857372dbcf6 100644 --- a/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardStream.Decompress.cs +++ b/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardStream.Decompress.cs @@ -196,10 +196,8 @@ public override async ValueTask ReadAsync(Memory buffer, Cancellation OperationStatus lastResult; while (!TryDecompress(buffer.Span, out bytesWritten, out lastResult)) { - if (_buffer.AvailableLength == 0) - { - _buffer.EnsureAvailableSpace(1); - } + _buffer.EnsureAvailableSpace(1); + int bytesRead = await _stream.ReadAsync(_buffer.AvailableMemory, cancellationToken).ConfigureAwait(false); if (bytesRead <= 0) { From 073b18e9972fd5900f9135b35cb94a23ba44c172 Mon Sep 17 00:00:00 2001 From: pumpkin-bit Date: Sat, 18 Apr 2026 19:22:02 +0500 Subject: [PATCH 4/9] Update src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardStream.Decompress.cs Co-authored-by: Miha Zupan --- .../IO/Compression/Zstandard/ZstandardStream.Decompress.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardStream.Decompress.cs b/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardStream.Decompress.cs index 8e1857372dbcf6..12f53f0ecf3ee8 100644 --- a/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardStream.Decompress.cs +++ b/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardStream.Decompress.cs @@ -117,10 +117,8 @@ public override int Read(Span buffer) OperationStatus lastResult; while (!TryDecompress(buffer, out bytesWritten, out lastResult)) { - if (_buffer.AvailableLength == 0) - { - _buffer.EnsureAvailableSpace(1); - } + _buffer.EnsureAvailableSpace(1); + int bytesRead = _stream.Read(_buffer.AvailableSpan); if (bytesRead <= 0) { From 7351c32fb2676e91e4003008fde4056255c329cc Mon Sep 17 00:00:00 2001 From: pumpkin-bit Date: Sat, 18 Apr 2026 19:47:10 +0500 Subject: [PATCH 5/9] add: unit test for ZstandardStream boundary condition --- .../CompressionStreamUnitTests.Zstandard.cs | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/libraries/System.IO.Compression/tests/Zstandard/CompressionStreamUnitTests.Zstandard.cs b/src/libraries/System.IO.Compression/tests/Zstandard/CompressionStreamUnitTests.Zstandard.cs index aa6f961de3d77b..444685d23d3f24 100644 --- a/src/libraries/System.IO.Compression/tests/Zstandard/CompressionStreamUnitTests.Zstandard.cs +++ b/src/libraries/System.IO.Compression/tests/Zstandard/CompressionStreamUnitTests.Zstandard.cs @@ -282,5 +282,42 @@ public void StreamTruncation_IsDetected(TestScenario testScenario) }, testScenario.ToString()).Dispose(); } + [Fact] + public async Task ZstandardStream_ArrayBuffer_Boundary_DoesNotTruncate() + { + // compress enough data to fill the 64kb ArrayBuffer + byte[] testData = ZstandardTestUtils.CreateTestData(150000); + using MemoryStream compressedStream = new(); + using (ZstandardStream compressor = new(compressedStream, CompressionLevel.Fastest, leaveOpen: true)) + { + compressor.Write(testData); + } + // read using a stream that drips small chunks e.g., 2000 bytes + // so that _availableStart gradually reaches the 65536 bounds and is forced to trigger + using DripStream dripStream = new(compressedStream.ToArray(), 2000); + using ZstandardStream decompressor = new(dripStream, CompressionMode.Decompress); + using MemoryStream decompressedStream = new(); + + await decompressor.CopyToAsync(decompressedStream); + Assert.Equal(testData, decompressedStream.ToArray()); + } + + private class DripStream : MemoryStream + { + private readonly int _dripSize; + public DripStream(byte[] buffer, int dripSize) : base(buffer) { _dripSize = dripSize; } + + public override int Read(byte[] buffer, int offset, int count) => + base.Read(buffer, offset, Math.Min(count, _dripSize)); + + public override int Read(Span buffer) => + base.Read(buffer.Slice(0, Math.Min(buffer.Length, _dripSize))); + + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => + base.ReadAsync(buffer, offset, Math.Min(count, _dripSize), cancellationToken); + + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) => + base.ReadAsync(buffer.Slice(0, Math.Min(buffer.Length, _dripSize)), cancellationToken); + } } } From 87bdf4ec501520d9b1488ae615912f4d80493a02 Mon Sep 17 00:00:00 2001 From: pumpkin-bit Date: Sat, 18 Apr 2026 20:28:19 +0500 Subject: [PATCH 6/9] add: system.threading for async support in tests --- .../CompressionStreamUnitTests.Zstandard.cs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/libraries/System.IO.Compression/tests/Zstandard/CompressionStreamUnitTests.Zstandard.cs b/src/libraries/System.IO.Compression/tests/Zstandard/CompressionStreamUnitTests.Zstandard.cs index 444685d23d3f24..57fde198b39aad 100644 --- a/src/libraries/System.IO.Compression/tests/Zstandard/CompressionStreamUnitTests.Zstandard.cs +++ b/src/libraries/System.IO.Compression/tests/Zstandard/CompressionStreamUnitTests.Zstandard.cs @@ -3,6 +3,7 @@ using System.Buffers; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Microsoft.DotNet.RemoteExecutor; using Xunit.Sdk; @@ -294,30 +295,28 @@ public async Task ZstandardStream_ArrayBuffer_Boundary_DoesNotTruncate() } // read using a stream that drips small chunks e.g., 2000 bytes // so that _availableStart gradually reaches the 65536 bounds and is forced to trigger + // EnsureAvailableSpace. using DripStream dripStream = new(compressedStream.ToArray(), 2000); using ZstandardStream decompressor = new(dripStream, CompressionMode.Decompress); using MemoryStream decompressedStream = new(); - + await decompressor.CopyToAsync(decompressedStream); Assert.Equal(testData, decompressedStream.ToArray()); } - + private class DripStream : MemoryStream { private readonly int _dripSize; public DripStream(byte[] buffer, int dripSize) : base(buffer) { _dripSize = dripSize; } - public override int Read(byte[] buffer, int offset, int count) => - base.Read(buffer, offset, Math.Min(count, _dripSize)); - + base.Read(buffer, offset, Math.Min(count, _dripSize)); public override int Read(Span buffer) => base.Read(buffer.Slice(0, Math.Min(buffer.Length, _dripSize))); - public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => base.ReadAsync(buffer, offset, Math.Min(count, _dripSize), cancellationToken); - public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) => base.ReadAsync(buffer.Slice(0, Math.Min(buffer.Length, _dripSize)), cancellationToken); } + } } From 595f9bb7f53c2015adb00995b517dc5c9108f742 Mon Sep 17 00:00:00 2001 From: pumpkin-bit Date: Mon, 20 Apr 2026 20:54:18 +0500 Subject: [PATCH 7/9] Delete src/libraries/System.IO.Compression/tests/Zstandard/CompressionStreamUnitTests.Zstandard.cs --- .../CompressionStreamUnitTests.Zstandard.cs | 322 ------------------ 1 file changed, 322 deletions(-) delete mode 100644 src/libraries/System.IO.Compression/tests/Zstandard/CompressionStreamUnitTests.Zstandard.cs diff --git a/src/libraries/System.IO.Compression/tests/Zstandard/CompressionStreamUnitTests.Zstandard.cs b/src/libraries/System.IO.Compression/tests/Zstandard/CompressionStreamUnitTests.Zstandard.cs deleted file mode 100644 index 57fde198b39aad..00000000000000 --- a/src/libraries/System.IO.Compression/tests/Zstandard/CompressionStreamUnitTests.Zstandard.cs +++ /dev/null @@ -1,322 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Buffers; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.DotNet.RemoteExecutor; -using Xunit.Sdk; -using Xunit; - -namespace System.IO.Compression -{ - public class ZstandardStreamUnitTests : CompressionStreamUnitTestBase - { - public override Stream CreateStream(Stream stream, CompressionMode mode) => new ZstandardStream(stream, mode); - public override Stream CreateStream(Stream stream, CompressionMode mode, bool leaveOpen) => new ZstandardStream(stream, mode, leaveOpen); - public override Stream CreateStream(Stream stream, CompressionLevel level) - { - if (PlatformDetection.Is32BitProcess && level == CompressionLevel.SmallestSize) - { - // Zstandard smallest size requires too much working memory - // (800+ MB) and causes intermittent allocation errors on 32-bit - // processes in CI. - level = CompressionLevel.Optimal; - } - - return new ZstandardStream(stream, level); - } - public override Stream CreateStream(Stream stream, CompressionLevel level, bool leaveOpen) - { - if (PlatformDetection.Is32BitProcess && level == CompressionLevel.SmallestSize) - { - // Zstandard smallest size requires too much working memory - // (800+ MB) and causes intermittent allocation errors on 32-bit - // processes in CI. - level = CompressionLevel.Optimal; - } - - return new ZstandardStream(stream, level, leaveOpen); - } - public override Stream CreateStream(Stream stream, ZLibCompressionOptions options, bool leaveOpen) => - new ZstandardStream(stream, options == null ? null : new ZstandardCompressionOptions { Quality = options.CompressionLevel }, leaveOpen); - - public override Stream BaseStream(Stream stream) => ((ZstandardStream)stream).BaseStream; - - // The tests are relying on an implementation detail of ZstandardStream, using knowledge of its internal buffer size - // in various test calculations. Currently the implementation is using the ArrayPool, which will round up to a - // power-of-2. If the buffer size employed changes (which could also mean that ArrayPool.Shared starts giving - // out different array sizes), the tests will need to be tweaked. - public override int BufferSize => 1 << 16; - - protected override string CompressedTestFile(string uncompressedPath) => Path.Combine("ZstandardTestData", Path.GetFileName(uncompressedPath) + ".zst"); - - [Fact] - public void ZstandardStream_WithEncoder_CompressesData() - { - ZstandardEncoder encoder = new(5, 10); - byte[] testData = ZstandardTestUtils.CreateTestData(); - using MemoryStream input = new(testData); - using MemoryStream output = new(); - - using (ZstandardStream compressionStream = new(output, encoder, leaveOpen: true)) - { - input.CopyTo(compressionStream); - } - - // Verify data was compressed - Assert.True(output.Length > 0); - Assert.True(output.Length < testData.Length); - - // Verify the encoder was reset, not disposed (should be reusable) - using MemoryStream output2 = new(); - using (ZstandardStream compressionStream2 = new(output2, encoder, leaveOpen: true)) - { - input.Position = 0; - input.CopyTo(compressionStream2); - } - - Assert.True(output2.Length > 0); - encoder.Dispose(); // Clean up - } - - [Fact] - public void ZstandardStream_WithDecoder_DecompressesData() - { - // First, create some compressed data - byte[] testData = ZstandardTestUtils.CreateTestData(); - byte[] compressedData = new byte[ZstandardEncoder.GetMaxCompressedLength(testData.Length)]; - bool compressResult = ZstandardEncoder.TryCompress(testData, compressedData, out int compressedLength); - Assert.True(compressResult); - - Array.Resize(ref compressedData, compressedLength); - - ZstandardDecoder decoder = new(); - using MemoryStream input = new(compressedData); - using MemoryStream output = new(); - - using (ZstandardStream decompressionStream = new(input, decoder, leaveOpen: true)) - { - decompressionStream.CopyTo(output); - } - - // Verify data was decompressed correctly - Assert.Equal(testData, output.ToArray()); - - // Verify the decoder was reset, not disposed (should be reusable) - using MemoryStream output2 = new(); - using (ZstandardStream decompressionStream2 = new(input, decoder, leaveOpen: true)) - { - input.Position = 0; - decompressionStream2.CopyTo(output2); - } - - Assert.Equal(testData, output2.ToArray()); - decoder.Dispose(); // Clean up - } - - [Theory] - [InlineData(true, -1)] - [InlineData(false, -1)] - [InlineData(true, 2)] - [InlineData(false, 2)] - public async Task ZstandardStream_SetSourceLength_SizeDiffers_InvalidDataException(bool async, long delta) - { - byte[] testData = ZstandardTestUtils.CreateTestData(); - using MemoryStream output = new(); - ZstandardStream compressionStream = new(output, CompressionLevel.Optimal); - - compressionStream.SetSourceLength(testData.Length + delta); - await Assert.ThrowsAsync(async () => - { - // for shorter source length, the error occurs during Write/WriteAsync - // for longer source length, the error occurs as part of Dispose/DisposeAsync - if (async) - { - await compressionStream.WriteAsync(testData, 0, testData.Length); - await compressionStream.DisposeAsync(); - } - else - { - compressionStream.Write(testData, 0, testData.Length); - compressionStream.Dispose(); - } - }); - } - - [Fact] - public void ZstandardStream_DecompressInvalidData_InvalidDataException() - { - byte[] invalidCompressedData = new byte[] { 0x01, 0x02, 0x03, 0x04, 0x05 }; - using MemoryStream input = new(invalidCompressedData); - using ZstandardStream decompressionStream = new(input, CompressionMode.Decompress); - byte[] buffer = new byte[16]; - - Assert.Throws(() => decompressionStream.Read(buffer, 0, buffer.Length)); - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task ZstandardStream_Roundtrip_WithDictionary(bool async) - { - byte[] dictionaryData = ZstandardTestUtils.CreateSampleDictionary(); - using ZstandardDictionary dictionary = ZstandardDictionary.Create(dictionaryData); - - byte[] testData = ZstandardTestUtils.CreateTestData(5000); - - using MemoryStream compressedStream = new(); - using (ZstandardStream compressionStream = new(compressedStream, CompressionMode.Compress, dictionary, leaveOpen: true)) - { - if (async) - { - await compressionStream.WriteAsync(testData, 0, testData.Length); - } - else - { - compressionStream.Write(testData, 0, testData.Length); - } - } - - compressedStream.Position = 0; - - using MemoryStream decompressedStream = new(); - using (ZstandardStream decompressionStream = new(compressedStream, CompressionMode.Decompress, dictionary)) - { - if (async) - { - await decompressionStream.CopyToAsync(decompressedStream); - } - else - { - decompressionStream.CopyTo(decompressedStream); - } - } - - Assert.Equal(testData, decompressedStream.ToArray()); - } - - [InlineData(TestScenario.ReadAsync)] - [InlineData(TestScenario.Read)] - [InlineData(TestScenario.Copy)] - [InlineData(TestScenario.CopyAsync)] - [InlineData(TestScenario.ReadByte)] - [InlineData(TestScenario.ReadByteAsync)] - [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - public void StreamTruncation_IsDetected(TestScenario testScenario) - { - RemoteExecutor.Invoke(async (testScenario) => - { - TestScenario scenario = Enum.Parse(testScenario); - - AppContext.SetSwitch("System.IO.Compression.UseStrictValidation", true); - - var buffer = new byte[16]; - byte[] source = Enumerable.Range(0, 64).Select(i => (byte)i).ToArray(); - byte[] compressedData; - using (var compressed = new MemoryStream()) - using (Stream compressor = CreateStream(compressed, CompressionMode.Compress)) - { - foreach (byte b in source) - { - compressor.WriteByte(b); - } - - compressor.Dispose(); - compressedData = compressed.ToArray(); - } - - for (var i = 1; i <= compressedData.Length; i += 1) - { - bool expectException = i < compressedData.Length; - using (var compressedStream = new MemoryStream(compressedData.Take(i).ToArray())) - { - using (Stream decompressor = CreateStream(compressedStream, CompressionMode.Decompress)) - { - var decompressedStream = new MemoryStream(); - - try - { - switch (scenario) - { - case TestScenario.Copy: - decompressor.CopyTo(decompressedStream); - break; - - case TestScenario.CopyAsync: - await decompressor.CopyToAsync(decompressedStream); - break; - - case TestScenario.Read: - while (decompressor.Read(buffer, 0, buffer.Length) != 0) { } - break; - - case TestScenario.ReadAsync: - while (await decompressor.ReadAsync(buffer, 0, buffer.Length) != 0) { } - break; - - case TestScenario.ReadByte: - while (decompressor.ReadByte() != -1) { } - break; - - case TestScenario.ReadByteAsync: - while (await decompressor.ReadByteAsync() != -1) { } - break; - } - } - catch (InvalidDataException e) - { - if (expectException) - continue; - - throw new XunitException($"An unexpected error occurred while decompressing data:{e}"); - } - - if (expectException) - { - throw new XunitException($"Truncated stream was decompressed successfully but exception was expected: length={i}/{compressedData.Length}"); - } - } - } - } - }, testScenario.ToString()).Dispose(); - } - - [Fact] - public async Task ZstandardStream_ArrayBuffer_Boundary_DoesNotTruncate() - { - // compress enough data to fill the 64kb ArrayBuffer - byte[] testData = ZstandardTestUtils.CreateTestData(150000); - using MemoryStream compressedStream = new(); - using (ZstandardStream compressor = new(compressedStream, CompressionLevel.Fastest, leaveOpen: true)) - { - compressor.Write(testData); - } - // read using a stream that drips small chunks e.g., 2000 bytes - // so that _availableStart gradually reaches the 65536 bounds and is forced to trigger - // EnsureAvailableSpace. - using DripStream dripStream = new(compressedStream.ToArray(), 2000); - using ZstandardStream decompressor = new(dripStream, CompressionMode.Decompress); - using MemoryStream decompressedStream = new(); - - await decompressor.CopyToAsync(decompressedStream); - Assert.Equal(testData, decompressedStream.ToArray()); - } - - private class DripStream : MemoryStream - { - private readonly int _dripSize; - public DripStream(byte[] buffer, int dripSize) : base(buffer) { _dripSize = dripSize; } - public override int Read(byte[] buffer, int offset, int count) => - base.Read(buffer, offset, Math.Min(count, _dripSize)); - public override int Read(Span buffer) => - base.Read(buffer.Slice(0, Math.Min(buffer.Length, _dripSize))); - public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => - base.ReadAsync(buffer, offset, Math.Min(count, _dripSize), cancellationToken); - public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) => - base.ReadAsync(buffer.Slice(0, Math.Min(buffer.Length, _dripSize)), cancellationToken); - } - - } -} From c18529bfbdbb0aef0e84dafbae4d9f002f260d5c Mon Sep 17 00:00:00 2001 From: pumpkin-bit Date: Mon, 20 Apr 2026 21:10:08 +0500 Subject: [PATCH 8/9] recovery CompressionStreamUnitTests.Zstandard.cs --- .../CompressionStreamUnitTests.Zstandard.cs | 322 ++++++++++++++++++ 1 file changed, 322 insertions(+) create mode 100644 src/libraries/System.IO.Compression/tests/Zstandard/CompressionStreamUnitTests.Zstandard.cs diff --git a/src/libraries/System.IO.Compression/tests/Zstandard/CompressionStreamUnitTests.Zstandard.cs b/src/libraries/System.IO.Compression/tests/Zstandard/CompressionStreamUnitTests.Zstandard.cs new file mode 100644 index 00000000000000..57fde198b39aad --- /dev/null +++ b/src/libraries/System.IO.Compression/tests/Zstandard/CompressionStreamUnitTests.Zstandard.cs @@ -0,0 +1,322 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.DotNet.RemoteExecutor; +using Xunit.Sdk; +using Xunit; + +namespace System.IO.Compression +{ + public class ZstandardStreamUnitTests : CompressionStreamUnitTestBase + { + public override Stream CreateStream(Stream stream, CompressionMode mode) => new ZstandardStream(stream, mode); + public override Stream CreateStream(Stream stream, CompressionMode mode, bool leaveOpen) => new ZstandardStream(stream, mode, leaveOpen); + public override Stream CreateStream(Stream stream, CompressionLevel level) + { + if (PlatformDetection.Is32BitProcess && level == CompressionLevel.SmallestSize) + { + // Zstandard smallest size requires too much working memory + // (800+ MB) and causes intermittent allocation errors on 32-bit + // processes in CI. + level = CompressionLevel.Optimal; + } + + return new ZstandardStream(stream, level); + } + public override Stream CreateStream(Stream stream, CompressionLevel level, bool leaveOpen) + { + if (PlatformDetection.Is32BitProcess && level == CompressionLevel.SmallestSize) + { + // Zstandard smallest size requires too much working memory + // (800+ MB) and causes intermittent allocation errors on 32-bit + // processes in CI. + level = CompressionLevel.Optimal; + } + + return new ZstandardStream(stream, level, leaveOpen); + } + public override Stream CreateStream(Stream stream, ZLibCompressionOptions options, bool leaveOpen) => + new ZstandardStream(stream, options == null ? null : new ZstandardCompressionOptions { Quality = options.CompressionLevel }, leaveOpen); + + public override Stream BaseStream(Stream stream) => ((ZstandardStream)stream).BaseStream; + + // The tests are relying on an implementation detail of ZstandardStream, using knowledge of its internal buffer size + // in various test calculations. Currently the implementation is using the ArrayPool, which will round up to a + // power-of-2. If the buffer size employed changes (which could also mean that ArrayPool.Shared starts giving + // out different array sizes), the tests will need to be tweaked. + public override int BufferSize => 1 << 16; + + protected override string CompressedTestFile(string uncompressedPath) => Path.Combine("ZstandardTestData", Path.GetFileName(uncompressedPath) + ".zst"); + + [Fact] + public void ZstandardStream_WithEncoder_CompressesData() + { + ZstandardEncoder encoder = new(5, 10); + byte[] testData = ZstandardTestUtils.CreateTestData(); + using MemoryStream input = new(testData); + using MemoryStream output = new(); + + using (ZstandardStream compressionStream = new(output, encoder, leaveOpen: true)) + { + input.CopyTo(compressionStream); + } + + // Verify data was compressed + Assert.True(output.Length > 0); + Assert.True(output.Length < testData.Length); + + // Verify the encoder was reset, not disposed (should be reusable) + using MemoryStream output2 = new(); + using (ZstandardStream compressionStream2 = new(output2, encoder, leaveOpen: true)) + { + input.Position = 0; + input.CopyTo(compressionStream2); + } + + Assert.True(output2.Length > 0); + encoder.Dispose(); // Clean up + } + + [Fact] + public void ZstandardStream_WithDecoder_DecompressesData() + { + // First, create some compressed data + byte[] testData = ZstandardTestUtils.CreateTestData(); + byte[] compressedData = new byte[ZstandardEncoder.GetMaxCompressedLength(testData.Length)]; + bool compressResult = ZstandardEncoder.TryCompress(testData, compressedData, out int compressedLength); + Assert.True(compressResult); + + Array.Resize(ref compressedData, compressedLength); + + ZstandardDecoder decoder = new(); + using MemoryStream input = new(compressedData); + using MemoryStream output = new(); + + using (ZstandardStream decompressionStream = new(input, decoder, leaveOpen: true)) + { + decompressionStream.CopyTo(output); + } + + // Verify data was decompressed correctly + Assert.Equal(testData, output.ToArray()); + + // Verify the decoder was reset, not disposed (should be reusable) + using MemoryStream output2 = new(); + using (ZstandardStream decompressionStream2 = new(input, decoder, leaveOpen: true)) + { + input.Position = 0; + decompressionStream2.CopyTo(output2); + } + + Assert.Equal(testData, output2.ToArray()); + decoder.Dispose(); // Clean up + } + + [Theory] + [InlineData(true, -1)] + [InlineData(false, -1)] + [InlineData(true, 2)] + [InlineData(false, 2)] + public async Task ZstandardStream_SetSourceLength_SizeDiffers_InvalidDataException(bool async, long delta) + { + byte[] testData = ZstandardTestUtils.CreateTestData(); + using MemoryStream output = new(); + ZstandardStream compressionStream = new(output, CompressionLevel.Optimal); + + compressionStream.SetSourceLength(testData.Length + delta); + await Assert.ThrowsAsync(async () => + { + // for shorter source length, the error occurs during Write/WriteAsync + // for longer source length, the error occurs as part of Dispose/DisposeAsync + if (async) + { + await compressionStream.WriteAsync(testData, 0, testData.Length); + await compressionStream.DisposeAsync(); + } + else + { + compressionStream.Write(testData, 0, testData.Length); + compressionStream.Dispose(); + } + }); + } + + [Fact] + public void ZstandardStream_DecompressInvalidData_InvalidDataException() + { + byte[] invalidCompressedData = new byte[] { 0x01, 0x02, 0x03, 0x04, 0x05 }; + using MemoryStream input = new(invalidCompressedData); + using ZstandardStream decompressionStream = new(input, CompressionMode.Decompress); + byte[] buffer = new byte[16]; + + Assert.Throws(() => decompressionStream.Read(buffer, 0, buffer.Length)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ZstandardStream_Roundtrip_WithDictionary(bool async) + { + byte[] dictionaryData = ZstandardTestUtils.CreateSampleDictionary(); + using ZstandardDictionary dictionary = ZstandardDictionary.Create(dictionaryData); + + byte[] testData = ZstandardTestUtils.CreateTestData(5000); + + using MemoryStream compressedStream = new(); + using (ZstandardStream compressionStream = new(compressedStream, CompressionMode.Compress, dictionary, leaveOpen: true)) + { + if (async) + { + await compressionStream.WriteAsync(testData, 0, testData.Length); + } + else + { + compressionStream.Write(testData, 0, testData.Length); + } + } + + compressedStream.Position = 0; + + using MemoryStream decompressedStream = new(); + using (ZstandardStream decompressionStream = new(compressedStream, CompressionMode.Decompress, dictionary)) + { + if (async) + { + await decompressionStream.CopyToAsync(decompressedStream); + } + else + { + decompressionStream.CopyTo(decompressedStream); + } + } + + Assert.Equal(testData, decompressedStream.ToArray()); + } + + [InlineData(TestScenario.ReadAsync)] + [InlineData(TestScenario.Read)] + [InlineData(TestScenario.Copy)] + [InlineData(TestScenario.CopyAsync)] + [InlineData(TestScenario.ReadByte)] + [InlineData(TestScenario.ReadByteAsync)] + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + public void StreamTruncation_IsDetected(TestScenario testScenario) + { + RemoteExecutor.Invoke(async (testScenario) => + { + TestScenario scenario = Enum.Parse(testScenario); + + AppContext.SetSwitch("System.IO.Compression.UseStrictValidation", true); + + var buffer = new byte[16]; + byte[] source = Enumerable.Range(0, 64).Select(i => (byte)i).ToArray(); + byte[] compressedData; + using (var compressed = new MemoryStream()) + using (Stream compressor = CreateStream(compressed, CompressionMode.Compress)) + { + foreach (byte b in source) + { + compressor.WriteByte(b); + } + + compressor.Dispose(); + compressedData = compressed.ToArray(); + } + + for (var i = 1; i <= compressedData.Length; i += 1) + { + bool expectException = i < compressedData.Length; + using (var compressedStream = new MemoryStream(compressedData.Take(i).ToArray())) + { + using (Stream decompressor = CreateStream(compressedStream, CompressionMode.Decompress)) + { + var decompressedStream = new MemoryStream(); + + try + { + switch (scenario) + { + case TestScenario.Copy: + decompressor.CopyTo(decompressedStream); + break; + + case TestScenario.CopyAsync: + await decompressor.CopyToAsync(decompressedStream); + break; + + case TestScenario.Read: + while (decompressor.Read(buffer, 0, buffer.Length) != 0) { } + break; + + case TestScenario.ReadAsync: + while (await decompressor.ReadAsync(buffer, 0, buffer.Length) != 0) { } + break; + + case TestScenario.ReadByte: + while (decompressor.ReadByte() != -1) { } + break; + + case TestScenario.ReadByteAsync: + while (await decompressor.ReadByteAsync() != -1) { } + break; + } + } + catch (InvalidDataException e) + { + if (expectException) + continue; + + throw new XunitException($"An unexpected error occurred while decompressing data:{e}"); + } + + if (expectException) + { + throw new XunitException($"Truncated stream was decompressed successfully but exception was expected: length={i}/{compressedData.Length}"); + } + } + } + } + }, testScenario.ToString()).Dispose(); + } + + [Fact] + public async Task ZstandardStream_ArrayBuffer_Boundary_DoesNotTruncate() + { + // compress enough data to fill the 64kb ArrayBuffer + byte[] testData = ZstandardTestUtils.CreateTestData(150000); + using MemoryStream compressedStream = new(); + using (ZstandardStream compressor = new(compressedStream, CompressionLevel.Fastest, leaveOpen: true)) + { + compressor.Write(testData); + } + // read using a stream that drips small chunks e.g., 2000 bytes + // so that _availableStart gradually reaches the 65536 bounds and is forced to trigger + // EnsureAvailableSpace. + using DripStream dripStream = new(compressedStream.ToArray(), 2000); + using ZstandardStream decompressor = new(dripStream, CompressionMode.Decompress); + using MemoryStream decompressedStream = new(); + + await decompressor.CopyToAsync(decompressedStream); + Assert.Equal(testData, decompressedStream.ToArray()); + } + + private class DripStream : MemoryStream + { + private readonly int _dripSize; + public DripStream(byte[] buffer, int dripSize) : base(buffer) { _dripSize = dripSize; } + public override int Read(byte[] buffer, int offset, int count) => + base.Read(buffer, offset, Math.Min(count, _dripSize)); + public override int Read(Span buffer) => + base.Read(buffer.Slice(0, Math.Min(buffer.Length, _dripSize))); + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => + base.ReadAsync(buffer, offset, Math.Min(count, _dripSize), cancellationToken); + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) => + base.ReadAsync(buffer.Slice(0, Math.Min(buffer.Length, _dripSize)), cancellationToken); + } + + } +} From bb339c23e6802327c3c973f8dd0ba9a27cea9d13 Mon Sep 17 00:00:00 2001 From: pumpkin-bit Date: Mon, 20 Apr 2026 21:12:31 +0500 Subject: [PATCH 9/9] Remove ZstandardStream boundary truncation test --- .../CompressionStreamUnitTests.Zstandard.cs | 36 ------------------- 1 file changed, 36 deletions(-) diff --git a/src/libraries/System.IO.Compression/tests/Zstandard/CompressionStreamUnitTests.Zstandard.cs b/src/libraries/System.IO.Compression/tests/Zstandard/CompressionStreamUnitTests.Zstandard.cs index 57fde198b39aad..aa6f961de3d77b 100644 --- a/src/libraries/System.IO.Compression/tests/Zstandard/CompressionStreamUnitTests.Zstandard.cs +++ b/src/libraries/System.IO.Compression/tests/Zstandard/CompressionStreamUnitTests.Zstandard.cs @@ -3,7 +3,6 @@ using System.Buffers; using System.Linq; -using System.Threading; using System.Threading.Tasks; using Microsoft.DotNet.RemoteExecutor; using Xunit.Sdk; @@ -283,40 +282,5 @@ public void StreamTruncation_IsDetected(TestScenario testScenario) }, testScenario.ToString()).Dispose(); } - [Fact] - public async Task ZstandardStream_ArrayBuffer_Boundary_DoesNotTruncate() - { - // compress enough data to fill the 64kb ArrayBuffer - byte[] testData = ZstandardTestUtils.CreateTestData(150000); - using MemoryStream compressedStream = new(); - using (ZstandardStream compressor = new(compressedStream, CompressionLevel.Fastest, leaveOpen: true)) - { - compressor.Write(testData); - } - // read using a stream that drips small chunks e.g., 2000 bytes - // so that _availableStart gradually reaches the 65536 bounds and is forced to trigger - // EnsureAvailableSpace. - using DripStream dripStream = new(compressedStream.ToArray(), 2000); - using ZstandardStream decompressor = new(dripStream, CompressionMode.Decompress); - using MemoryStream decompressedStream = new(); - - await decompressor.CopyToAsync(decompressedStream); - Assert.Equal(testData, decompressedStream.ToArray()); - } - - private class DripStream : MemoryStream - { - private readonly int _dripSize; - public DripStream(byte[] buffer, int dripSize) : base(buffer) { _dripSize = dripSize; } - public override int Read(byte[] buffer, int offset, int count) => - base.Read(buffer, offset, Math.Min(count, _dripSize)); - public override int Read(Span buffer) => - base.Read(buffer.Slice(0, Math.Min(buffer.Length, _dripSize))); - public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => - base.ReadAsync(buffer, offset, Math.Min(count, _dripSize), cancellationToken); - public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) => - base.ReadAsync(buffer.Slice(0, Math.Min(buffer.Length, _dripSize)), cancellationToken); - } - } }